{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Train Model\n",
    "\n",
    "This notebook is used for following steps:\n",
    "\n",
    "1. Model Training and Validation\n",
    "2. Plotting loss history over epochs\n",
    "\n",
    "**1. Import Required Libraries:-** "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "from torchvision import transforms\n",
    "import torch\n",
    "import torch.optim as optim\n",
    "from torch.utils.data import random_split, DataLoader\n",
    "import time\n",
    "import cv2\n",
    "import numpy as np\n",
    "import os\n",
    "import matplotlib.pyplot as plt\n",
    "#Local Imports\n",
    "from transforms import Normalize,SequenceRandomTransform,ToTensor\n",
    "from dataset import HeadposeDataset, DatasetFromSubset\n",
    "from model import FSANet\n",
    "\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**2. Compose Augmentation Transform and Create Dataset:-**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "x (images) shape:  (122415, 64, 64, 3)\n",
      "y (poses) shape:  (122415, 3)\n"
     ]
    }
   ],
   "source": [
    "train_transform = transforms.Compose([\n",
    "            SequenceRandomTransform(),\n",
    "            Normalize(mean=127.5,std=128),\n",
    "            ToTensor()\n",
    "            ])\n",
    "\n",
    "validation_transform = transforms.Compose([Normalize(mean=127.5,std=128),ToTensor()])\n",
    "\n",
    "data_path = '../data/type1/train'\n",
    "\n",
    "hdb = HeadposeDataset(data_path,transform=None)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**3. Create Train-Validation Split from Dataset:-**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Split dataset into train and validation\n",
    "train_size = int(0.8 * len(hdb))\n",
    "validation_size = len(hdb) - train_size\n",
    "train_subset, validation_subset = random_split(hdb, [train_size, validation_size])\n",
    "\n",
    "train_dataset = DatasetFromSubset(train_subset,train_transform)\n",
    "validation_dataset = DatasetFromSubset(validation_subset,validation_transform)\n",
    "\n",
    "del hdb,train_subset,validation_subset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train Dataset Length:  97932\n",
      "Validation Dataset Length:  24483\n"
     ]
    }
   ],
   "source": [
    "print('Train Dataset Length: ',len(train_dataset))\n",
    "print('Validation Dataset Length: ',len(validation_dataset))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**4. Create Dataloaders for Train and Validation set:-**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Setup dataloaders for train and validation\n",
    "batch_size = 16\n",
    "\n",
    "train_loader = DataLoader(train_dataset, \n",
    "                          batch_size=batch_size,\n",
    "                          shuffle=True)\n",
    "\n",
    "validation_loader = DataLoader(validation_dataset, \n",
    "                          batch_size=batch_size,\n",
    "                          shuffle=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**5. Define Model Function:-**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "model = FSANet(var=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**6. Define Loss and Optimizer:-**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "criterion = torch.nn.L1Loss()\n",
    "learning_rate = 0.001\n",
    "optimizer = optim.AdamW(model.parameters(),\n",
    "                       lr=learning_rate)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**7. define learning rate decay scheduler:-**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "scheduler = optim.lr_scheduler.StepLR(optimizer, step_size=30, gamma=0.1)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**8. Place Model in GPU:-**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "FSANet(\n",
       "  (msms): MultiStreamMultiStage(\n",
       "    (avgpool): AvgPool2d(kernel_size=2, stride=2, padding=0)\n",
       "    (s0_conv0): SepConvBlock(\n",
       "      (conv): SepConv2d(\n",
       "        (depthwise): Conv2d(3, 3, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=3)\n",
       "        (pointwise): Conv2d(3, 16, kernel_size=(1, 1), stride=(1, 1))\n",
       "      )\n",
       "      (bn): BatchNorm2d(16, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      (act): ReLU()\n",
       "    )\n",
       "    (s0_conv1_0): SepConvBlock(\n",
       "      (conv): SepConv2d(\n",
       "        (depthwise): Conv2d(16, 16, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=16)\n",
       "        (pointwise): Conv2d(16, 32, kernel_size=(1, 1), stride=(1, 1))\n",
       "      )\n",
       "      (bn): BatchNorm2d(32, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      (act): ReLU()\n",
       "    )\n",
       "    (s0_conv1_1): SepConvBlock(\n",
       "      (conv): SepConv2d(\n",
       "        (depthwise): Conv2d(32, 32, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=32)\n",
       "        (pointwise): Conv2d(32, 32, kernel_size=(1, 1), stride=(1, 1))\n",
       "      )\n",
       "      (bn): BatchNorm2d(32, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      (act): ReLU()\n",
       "    )\n",
       "    (s0_conv1_out): Conv2dAct(\n",
       "      (conv): Conv2d(32, 64, kernel_size=(1, 1), stride=(1, 1))\n",
       "      (act): ReLU()\n",
       "    )\n",
       "    (s0_conv2_0): SepConvBlock(\n",
       "      (conv): SepConv2d(\n",
       "        (depthwise): Conv2d(32, 32, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=32)\n",
       "        (pointwise): Conv2d(32, 64, kernel_size=(1, 1), stride=(1, 1))\n",
       "      )\n",
       "      (bn): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      (act): ReLU()\n",
       "    )\n",
       "    (s0_conv2_1): SepConvBlock(\n",
       "      (conv): SepConv2d(\n",
       "        (depthwise): Conv2d(64, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=64)\n",
       "        (pointwise): Conv2d(64, 64, kernel_size=(1, 1), stride=(1, 1))\n",
       "      )\n",
       "      (bn): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      (act): ReLU()\n",
       "    )\n",
       "    (s0_conv2_out): Conv2dAct(\n",
       "      (conv): Conv2d(64, 64, kernel_size=(1, 1), stride=(1, 1))\n",
       "      (act): ReLU()\n",
       "    )\n",
       "    (s0_conv3_0): SepConvBlock(\n",
       "      (conv): SepConv2d(\n",
       "        (depthwise): Conv2d(64, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=64)\n",
       "        (pointwise): Conv2d(64, 128, kernel_size=(1, 1), stride=(1, 1))\n",
       "      )\n",
       "      (bn): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      (act): ReLU()\n",
       "    )\n",
       "    (s0_conv3_1): SepConvBlock(\n",
       "      (conv): SepConv2d(\n",
       "        (depthwise): Conv2d(128, 128, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=128)\n",
       "        (pointwise): Conv2d(128, 128, kernel_size=(1, 1), stride=(1, 1))\n",
       "      )\n",
       "      (bn): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      (act): ReLU()\n",
       "    )\n",
       "    (s0_conv3_out): Conv2dAct(\n",
       "      (conv): Conv2d(128, 64, kernel_size=(1, 1), stride=(1, 1))\n",
       "      (act): ReLU()\n",
       "    )\n",
       "    (maxpool): MaxPool2d(kernel_size=2, stride=2, padding=0, dilation=1, ceil_mode=False)\n",
       "    (s1_conv0): SepConvBlock(\n",
       "      (conv): SepConv2d(\n",
       "        (depthwise): Conv2d(3, 3, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=3)\n",
       "        (pointwise): Conv2d(3, 16, kernel_size=(1, 1), stride=(1, 1))\n",
       "      )\n",
       "      (bn): BatchNorm2d(16, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      (act): ReLU()\n",
       "    )\n",
       "    (s1_conv1_0): SepConvBlock(\n",
       "      (conv): SepConv2d(\n",
       "        (depthwise): Conv2d(16, 16, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=16)\n",
       "        (pointwise): Conv2d(16, 32, kernel_size=(1, 1), stride=(1, 1))\n",
       "      )\n",
       "      (bn): BatchNorm2d(32, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      (act): Tanh()\n",
       "    )\n",
       "    (s1_conv1_1): SepConvBlock(\n",
       "      (conv): SepConv2d(\n",
       "        (depthwise): Conv2d(32, 32, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=32)\n",
       "        (pointwise): Conv2d(32, 32, kernel_size=(1, 1), stride=(1, 1))\n",
       "      )\n",
       "      (bn): BatchNorm2d(32, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      (act): Tanh()\n",
       "    )\n",
       "    (s1_conv1_out): Conv2dAct(\n",
       "      (conv): Conv2d(32, 64, kernel_size=(1, 1), stride=(1, 1))\n",
       "      (act): Tanh()\n",
       "    )\n",
       "    (s1_conv2_0): SepConvBlock(\n",
       "      (conv): SepConv2d(\n",
       "        (depthwise): Conv2d(32, 32, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=32)\n",
       "        (pointwise): Conv2d(32, 64, kernel_size=(1, 1), stride=(1, 1))\n",
       "      )\n",
       "      (bn): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      (act): Tanh()\n",
       "    )\n",
       "    (s1_conv2_1): SepConvBlock(\n",
       "      (conv): SepConv2d(\n",
       "        (depthwise): Conv2d(64, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=64)\n",
       "        (pointwise): Conv2d(64, 64, kernel_size=(1, 1), stride=(1, 1))\n",
       "      )\n",
       "      (bn): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      (act): Tanh()\n",
       "    )\n",
       "    (s1_conv2_out): Conv2dAct(\n",
       "      (conv): Conv2d(64, 64, kernel_size=(1, 1), stride=(1, 1))\n",
       "      (act): Tanh()\n",
       "    )\n",
       "    (s1_conv3_0): SepConvBlock(\n",
       "      (conv): SepConv2d(\n",
       "        (depthwise): Conv2d(64, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=64)\n",
       "        (pointwise): Conv2d(64, 128, kernel_size=(1, 1), stride=(1, 1))\n",
       "      )\n",
       "      (bn): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      (act): Tanh()\n",
       "    )\n",
       "    (s1_conv3_1): SepConvBlock(\n",
       "      (conv): SepConv2d(\n",
       "        (depthwise): Conv2d(128, 128, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=128)\n",
       "        (pointwise): Conv2d(128, 128, kernel_size=(1, 1), stride=(1, 1))\n",
       "      )\n",
       "      (bn): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      (act): Tanh()\n",
       "    )\n",
       "    (s1_conv3_out): Conv2dAct(\n",
       "      (conv): Conv2d(128, 64, kernel_size=(1, 1), stride=(1, 1))\n",
       "      (act): Tanh()\n",
       "    )\n",
       "  )\n",
       "  (fgsm): FineGrainedStructureMapping(\n",
       "    (attention_maps): ScoringFunction(\n",
       "      (reduce_channel): Conv2dAct(\n",
       "        (conv): Conv2d(64, 1, kernel_size=(1, 1), stride=(1, 1))\n",
       "        (act): Sigmoid()\n",
       "      )\n",
       "    )\n",
       "    (fm): Linear(in_features=64, out_features=960, bias=True)\n",
       "    (fc): Linear(in_features=192, out_features=35, bias=True)\n",
       "  )\n",
       "  (caps_layer): CapsuleLayer1d()\n",
       "  (eaf): ExtractAggregatedFeatures()\n",
       "  (esp_s1): ExtractSSRParams(\n",
       "    (shift_fc): Linear(in_features=4, out_features=3, bias=True)\n",
       "    (scale_fc): Linear(in_features=4, out_features=3, bias=True)\n",
       "    (pred_fc): Linear(in_features=8, out_features=9, bias=True)\n",
       "  )\n",
       "  (esp_s2): ExtractSSRParams(\n",
       "    (shift_fc): Linear(in_features=4, out_features=3, bias=True)\n",
       "    (scale_fc): Linear(in_features=4, out_features=3, bias=True)\n",
       "    (pred_fc): Linear(in_features=8, out_features=9, bias=True)\n",
       "  )\n",
       "  (esp_s3): ExtractSSRParams(\n",
       "    (shift_fc): Linear(in_features=4, out_features=3, bias=True)\n",
       "    (scale_fc): Linear(in_features=4, out_features=3, bias=True)\n",
       "    (pred_fc): Linear(in_features=8, out_features=9, bias=True)\n",
       "  )\n",
       "  (ssr): SSRLayer()\n",
       ")"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "device = torch.device(\"cuda\")\n",
    "model.to(device)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**9. Define Training History Dictionary:-**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "history = {'train_loss' : [], 'validation_loss' : []}\n",
    "\n",
    "#this holds best states wrt validation loss that we are saving,\n",
    "#we can use these to resume from last best loss or keep the best one for inference later. \n",
    "best_states = {}\n",
    "\n",
    "#this is where we save our best state_dict during training and validation\n",
    "best_states['model'] = model.state_dict()\n",
    "\n",
    "#this is for storing optim state if we wish to continue training from our last best state\n",
    "best_states['optim'] = optimizer.state_dict()\n",
    "\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**10. Define Training and Validation Function:-**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "def train_net(n_epochs):\n",
    "    global best_states,history\n",
    "    \n",
    "    #average test loss over epoch used to find best model parameters\n",
    "    if(len(history['validation_loss'])>0):\n",
    "        min_loss_idx = np.argmin(history['validation_loss'])\n",
    "        best_loss = history['validation_loss'][min_loss_idx]\n",
    "    else:\n",
    "        best_loss = 10000\n",
    "    \n",
    "    #this is where we store all our losses during epoch before averaging them and storing in history\n",
    "    itter_loss = []  \n",
    "    t = time.time()\n",
    "    for epoch in range(n_epochs):  # loop over the dataset multiple times\n",
    "        # prepare the model for training\n",
    "        model.train()\n",
    "        \n",
    "        train_running_loss = []\n",
    "        \n",
    "        print(f'Epoch: {epoch}')\n",
    "        # train on batches of data, assumes you already have train_loader\n",
    "        print(\"Printing Train Loss...\")\n",
    "        for batch_i, data in enumerate(train_loader):\n",
    "            # get the input images and their corresponding poses\n",
    "            images,gt_poses = data\n",
    "            \n",
    "            # put data inside gpu\n",
    "            images = images.float().to(device)\n",
    "            gt_poses = gt_poses.float().to(device)\n",
    "\n",
    "            # call model forward pass\n",
    "            predicted_poses = model(images)\n",
    "\n",
    "            # calculate the softmax loss between predicted poses and ground truth poses\n",
    "            loss = criterion(predicted_poses, gt_poses)\n",
    "\n",
    "            # zero the parameter (weight) gradients\n",
    "            optimizer.zero_grad()\n",
    "            \n",
    "            # backward pass to calculate the weight gradients\n",
    "            loss.backward()\n",
    "\n",
    "            # update the weights\n",
    "            optimizer.step()\n",
    "            \n",
    "            # get loss\n",
    "            loss_scalar = loss.item()\n",
    "\n",
    "            #update loss logs\n",
    "            train_running_loss.append(loss_scalar)\n",
    "            \n",
    "            #collect batch losses to compute epoch loss later\n",
    "            itter_loss.append(loss_scalar)\n",
    "            \n",
    "            if batch_i % 1000 == 999:    # print every 100 batches\n",
    "                print(f'Batch: {batch_i+1}, Avg. Train Loss: {np.mean(train_running_loss)}')\n",
    "                train_running_loss = []\n",
    "        else: \n",
    "            history['train_loss'].append(np.mean(itter_loss))\n",
    "            itter_loss.clear()\n",
    "            validation_running_loss = []\n",
    "            model.eval()\n",
    "            with torch.no_grad():\n",
    "                print(\"Printing Validation Loss...\")\n",
    "                for batch_i, data in enumerate(validation_loader):\n",
    "                    # get the input images and their corresponding poses\n",
    "                    images,gt_poses = data\n",
    "\n",
    "                    # put data inside gpu\n",
    "                    images = images.float().to(device)\n",
    "                    gt_poses = gt_poses.float().to(device)\n",
    "\n",
    "                    # call model forward pass\n",
    "                    predicted_poses = model(images)\n",
    "\n",
    "                    # calculate the softmax loss between predicted poses and ground truth poses\n",
    "                    loss = criterion(predicted_poses, gt_poses)\n",
    "                    \n",
    "                    #convert loss into a scalar using .item()\n",
    "                    loss_scalar = loss.item()\n",
    "                    \n",
    "                    #add loss to the running_loss, use\n",
    "                    validation_running_loss.append(loss_scalar)\n",
    "                    \n",
    "                    #collect batch losses to compute epoch loss later\n",
    "                    itter_loss.append(loss_scalar)\n",
    "                    \n",
    "                    \n",
    "                    if batch_i % 1000 == 999:    # print every 10 batches\n",
    "                        print(f'Batch: {batch_i+1}, Avg. Validation Loss: {np.mean(validation_running_loss)}')\n",
    "                        validation_running_loss = []\n",
    "                        \n",
    "            history['validation_loss'].append(np.mean(itter_loss))\n",
    "            itter_loss.clear()\n",
    "            \n",
    "            #if current is better than previous, update state_dict and store current as best\n",
    "            if(history['validation_loss'][-1] < best_loss):\n",
    "                best_loss = history['validation_loss'][-1]\n",
    "                best_states['model'] = model.state_dict()\n",
    "                best_states['optim'] = optimizer.state_dict()\n",
    "                #Save Model Checkpoint\n",
    "                torch.save({\n",
    "                            'total_epochs': len(history['train_loss']),\n",
    "                            'best_states': best_states,\n",
    "                            'history' : history\n",
    "                            }, '../checkpoints/8_8_2020_temp2.chkpt')\n",
    "                \n",
    "                print(f'Model improved since last epoch! New Best Val Loss: {best_loss}')\n",
    "                \n",
    "        # update lr schedular\n",
    "        scheduler.step()                                     \n",
    "\n",
    "    print('Finished Training')\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**11. Train Model for 90 epochs:-**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 0\n",
      "Printing Train Loss...\n",
      "Batch: 1000, Avg. Train Loss: 12.751319241523742\n",
      "Batch: 2000, Avg. Train Loss: 9.951046291828156\n",
      "Batch: 3000, Avg. Train Loss: 9.19135228061676\n",
      "Batch: 4000, Avg. Train Loss: 8.485794114112855\n",
      "Batch: 5000, Avg. Train Loss: 8.005072618484498\n",
      "Batch: 6000, Avg. Train Loss: 7.66648928308487\n",
      "Printing Validation Loss...\n",
      "Batch: 1000, Avg. Validation Loss: 6.184938750028611\n",
      "Model improved since last epoch! New Best Val Loss: 6.200778093936005\n",
      "Epoch: 1\n",
      "Printing Train Loss...\n",
      "Batch: 1000, Avg. Train Loss: 7.435550868988037\n",
      "Batch: 2000, Avg. Train Loss: 7.160996512889862\n",
      "Batch: 3000, Avg. Train Loss: 6.886740171194076\n",
      "Batch: 4000, Avg. Train Loss: 6.813953850984573\n",
      "Batch: 5000, Avg. Train Loss: 6.628541767120361\n",
      "Batch: 6000, Avg. Train Loss: 6.429875091075897\n",
      "Printing Validation Loss...\n",
      "Batch: 1000, Avg. Validation Loss: 5.1074435946941374\n",
      "Model improved since last epoch! New Best Val Loss: 5.103827243689071\n",
      "Epoch: 2\n",
      "Printing Train Loss...\n",
      "Batch: 1000, Avg. Train Loss: 6.195648370742798\n",
      "Batch: 2000, Avg. Train Loss: 6.240176671743393\n",
      "Batch: 3000, Avg. Train Loss: 6.0133696556091305\n",
      "Batch: 4000, Avg. Train Loss: 5.955946234464645\n",
      "Batch: 5000, Avg. Train Loss: 5.930267371416092\n",
      "Batch: 6000, Avg. Train Loss: 5.887298090696335\n",
      "Printing Validation Loss...\n",
      "Batch: 1000, Avg. Validation Loss: 4.721472725629806\n",
      "Model improved since last epoch! New Best Val Loss: 4.719829767066463\n",
      "Epoch: 3\n",
      "Printing Train Loss...\n",
      "Batch: 1000, Avg. Train Loss: 5.626614264965057\n",
      "Batch: 2000, Avg. Train Loss: 5.602130374908447\n",
      "Batch: 3000, Avg. Train Loss: 5.522494968414307\n",
      "Batch: 4000, Avg. Train Loss: 5.497821833133697\n",
      "Batch: 5000, Avg. Train Loss: 5.421201204538345\n",
      "Batch: 6000, Avg. Train Loss: 5.353183519363403\n",
      "Printing Validation Loss...\n",
      "Batch: 1000, Avg. Validation Loss: 4.445078087091446\n",
      "Model improved since last epoch! New Best Val Loss: 4.44964891769464\n",
      "Epoch: 4\n",
      "Printing Train Loss...\n",
      "Batch: 1000, Avg. Train Loss: 5.319172201871872\n",
      "Batch: 2000, Avg. Train Loss: 5.2366302144527435\n",
      "Batch: 3000, Avg. Train Loss: 5.222496994256973\n",
      "Batch: 4000, Avg. Train Loss: 5.16297829580307\n",
      "Batch: 5000, Avg. Train Loss: 5.049959500551224\n",
      "Batch: 6000, Avg. Train Loss: 5.078923593997955\n",
      "Printing Validation Loss...\n",
      "Batch: 1000, Avg. Validation Loss: 4.118027767896653\n",
      "Model improved since last epoch! New Best Val Loss: 4.132470054925461\n",
      "Epoch: 5\n",
      "Printing Train Loss...\n",
      "Batch: 1000, Avg. Train Loss: 5.022735588312149\n",
      "Batch: 2000, Avg. Train Loss: 5.017530950069427\n",
      "Batch: 3000, Avg. Train Loss: 4.955632864952087\n",
      "Batch: 4000, Avg. Train Loss: 4.891472409486771\n",
      "Batch: 5000, Avg. Train Loss: 4.92680265712738\n",
      "Batch: 6000, Avg. Train Loss: 4.817568868160248\n",
      "Printing Validation Loss...\n",
      "Batch: 1000, Avg. Validation Loss: 3.9633943548202515\n",
      "Model improved since last epoch! New Best Val Loss: 3.978200660041545\n",
      "Epoch: 6\n",
      "Printing Train Loss...\n",
      "Batch: 1000, Avg. Train Loss: 4.773378051996231\n",
      "Batch: 2000, Avg. Train Loss: 4.750433032989502\n",
      "Batch: 3000, Avg. Train Loss: 4.800783493995667\n",
      "Batch: 4000, Avg. Train Loss: 4.705104313373566\n",
      "Batch: 5000, Avg. Train Loss: 4.712901577472687\n",
      "Batch: 6000, Avg. Train Loss: 4.840962108135224\n",
      "Printing Validation Loss...\n",
      "Batch: 1000, Avg. Validation Loss: 3.803403870820999\n",
      "Model improved since last epoch! New Best Val Loss: 3.8134138504716013\n",
      "Epoch: 7\n",
      "Printing Train Loss...\n",
      "Batch: 1000, Avg. Train Loss: 4.660885274410248\n",
      "Batch: 2000, Avg. Train Loss: 4.616836881637573\n",
      "Batch: 3000, Avg. Train Loss: 4.717553568124771\n",
      "Batch: 4000, Avg. Train Loss: 4.589278087854385\n",
      "Batch: 5000, Avg. Train Loss: 4.6268023760318755\n",
      "Batch: 6000, Avg. Train Loss: 4.568331587791443\n",
      "Printing Validation Loss...\n",
      "Batch: 1000, Avg. Validation Loss: 3.7808202142715452\n",
      "Model improved since last epoch! New Best Val Loss: 3.7847464699748294\n",
      "Epoch: 8\n",
      "Printing Train Loss...\n",
      "Batch: 1000, Avg. Train Loss: 4.482490998029709\n",
      "Batch: 2000, Avg. Train Loss: 4.573989012718201\n",
      "Batch: 3000, Avg. Train Loss: 4.572580801963806\n",
      "Batch: 4000, Avg. Train Loss: 4.483922099590301\n",
      "Batch: 5000, Avg. Train Loss: 4.5101837682724\n",
      "Batch: 6000, Avg. Train Loss: 4.428215290546417\n",
      "Printing Validation Loss...\n",
      "Batch: 1000, Avg. Validation Loss: 3.7065414175987246\n",
      "Model improved since last epoch! New Best Val Loss: 3.7192818128073784\n",
      "Epoch: 9\n",
      "Printing Train Loss...\n",
      "Batch: 1000, Avg. Train Loss: 4.4814321415424345\n",
      "Batch: 2000, Avg. Train Loss: 4.410909072160721\n",
      "Batch: 3000, Avg. Train Loss: 4.432186325550079\n",
      "Batch: 4000, Avg. Train Loss: 4.3635687325000765\n",
      "Batch: 5000, Avg. Train Loss: 4.401636340618134\n",
      "Batch: 6000, Avg. Train Loss: 4.402089354515076\n",
      "Printing Validation Loss...\n",
      "Batch: 1000, Avg. Validation Loss: 3.7178144624233247\n",
      "Epoch: 10\n",
      "Printing Train Loss...\n",
      "Batch: 1000, Avg. Train Loss: 4.349180361270904\n",
      "Batch: 2000, Avg. Train Loss: 4.355805153846741\n",
      "Batch: 3000, Avg. Train Loss: 4.316235120058059\n",
      "Batch: 4000, Avg. Train Loss: 4.312097276449204\n",
      "Batch: 5000, Avg. Train Loss: 4.3136701526641845\n",
      "Batch: 6000, Avg. Train Loss: 4.353901655435562\n",
      "Printing Validation Loss...\n",
      "Batch: 1000, Avg. Validation Loss: 3.6415532672405244\n",
      "Model improved since last epoch! New Best Val Loss: 3.6370559944038217\n",
      "Epoch: 11\n",
      "Printing Train Loss...\n",
      "Batch: 1000, Avg. Train Loss: 4.286971686840057\n",
      "Batch: 2000, Avg. Train Loss: 4.327780596733093\n",
      "Batch: 3000, Avg. Train Loss: 4.2885206758975984\n",
      "Batch: 4000, Avg. Train Loss: 4.29078473687172\n",
      "Batch: 5000, Avg. Train Loss: 4.230758675336838\n",
      "Batch: 6000, Avg. Train Loss: 4.204177791118622\n",
      "Printing Validation Loss...\n",
      "Batch: 1000, Avg. Validation Loss: 3.623711083173752\n",
      "Model improved since last epoch! New Best Val Loss: 3.626347221867233\n",
      "Epoch: 12\n",
      "Printing Train Loss...\n",
      "Batch: 1000, Avg. Train Loss: 4.246445865631103\n",
      "Batch: 2000, Avg. Train Loss: 4.169403271198273\n",
      "Batch: 3000, Avg. Train Loss: 4.265952890634536\n",
      "Batch: 4000, Avg. Train Loss: 4.255010341405868\n",
      "Batch: 5000, Avg. Train Loss: 4.148012310266495\n",
      "Batch: 6000, Avg. Train Loss: 4.205400464057923\n",
      "Printing Validation Loss...\n",
      "Batch: 1000, Avg. Validation Loss: 3.6040132796764373\n",
      "Model improved since last epoch! New Best Val Loss: 3.603986208147663\n",
      "Epoch: 13\n",
      "Printing Train Loss...\n",
      "Batch: 1000, Avg. Train Loss: 4.142011077165604\n",
      "Batch: 2000, Avg. Train Loss: 4.132070030212402\n",
      "Batch: 3000, Avg. Train Loss: 4.158402419805527\n",
      "Batch: 4000, Avg. Train Loss: 4.186225215435028\n",
      "Batch: 5000, Avg. Train Loss: 4.14023165845871\n",
      "Batch: 6000, Avg. Train Loss: 4.129270595312119\n",
      "Printing Validation Loss...\n",
      "Batch: 1000, Avg. Validation Loss: 3.5024901732206346\n",
      "Model improved since last epoch! New Best Val Loss: 3.5021042276409853\n",
      "Epoch: 14\n",
      "Printing Train Loss...\n",
      "Batch: 1000, Avg. Train Loss: 4.038510570049286\n",
      "Batch: 2000, Avg. Train Loss: 4.067742336511612\n",
      "Batch: 3000, Avg. Train Loss: 4.119893229007721\n",
      "Batch: 4000, Avg. Train Loss: 4.097747985601425\n",
      "Batch: 5000, Avg. Train Loss: 4.147823089838028\n",
      "Batch: 6000, Avg. Train Loss: 4.0794329688549045\n",
      "Printing Validation Loss...\n",
      "Batch: 1000, Avg. Validation Loss: 3.5915499472618104\n",
      "Epoch: 15\n",
      "Printing Train Loss...\n",
      "Batch: 1000, Avg. Train Loss: 4.144732920646668\n",
      "Batch: 2000, Avg. Train Loss: 4.026606845855713\n",
      "Batch: 3000, Avg. Train Loss: 4.108107289552689\n",
      "Batch: 4000, Avg. Train Loss: 4.045585798621178\n",
      "Batch: 5000, Avg. Train Loss: 4.103447109699249\n",
      "Batch: 6000, Avg. Train Loss: 3.9962116525173186\n",
      "Printing Validation Loss...\n",
      "Batch: 1000, Avg. Validation Loss: 3.3696983580589293\n",
      "Model improved since last epoch! New Best Val Loss: 3.3803891880227255\n",
      "Epoch: 16\n",
      "Printing Train Loss...\n",
      "Batch: 1000, Avg. Train Loss: 4.03588719534874\n",
      "Batch: 2000, Avg. Train Loss: 4.055564250946045\n",
      "Batch: 3000, Avg. Train Loss: 4.0486472187042235\n",
      "Batch: 4000, Avg. Train Loss: 3.970550538778305\n",
      "Batch: 5000, Avg. Train Loss: 4.0307148442268375\n",
      "Batch: 6000, Avg. Train Loss: 4.046828874230385\n",
      "Printing Validation Loss...\n",
      "Batch: 1000, Avg. Validation Loss: 3.420742074608803\n",
      "Epoch: 17\n",
      "Printing Train Loss...\n",
      "Batch: 1000, Avg. Train Loss: 3.9838535988330843\n",
      "Batch: 2000, Avg. Train Loss: 3.984585225343704\n",
      "Batch: 3000, Avg. Train Loss: 4.033516820430756\n",
      "Batch: 4000, Avg. Train Loss: 4.083494306087494\n",
      "Batch: 5000, Avg. Train Loss: 4.0139178874492645\n",
      "Batch: 6000, Avg. Train Loss: 3.9771955242156984\n",
      "Printing Validation Loss...\n",
      "Batch: 1000, Avg. Validation Loss: 3.4025206941366197\n",
      "Epoch: 18\n",
      "Printing Train Loss...\n",
      "Batch: 1000, Avg. Train Loss: 4.016831189632415\n",
      "Batch: 2000, Avg. Train Loss: 3.979005539178848\n",
      "Batch: 3000, Avg. Train Loss: 4.00186477971077\n",
      "Batch: 4000, Avg. Train Loss: 4.010084434747696\n",
      "Batch: 5000, Avg. Train Loss: 4.019560162067413\n",
      "Batch: 6000, Avg. Train Loss: 3.887311587333679\n",
      "Printing Validation Loss...\n",
      "Batch: 1000, Avg. Validation Loss: 3.3944841270446777\n",
      "Epoch: 19\n",
      "Printing Train Loss...\n",
      "Batch: 1000, Avg. Train Loss: 3.983113799571991\n",
      "Batch: 2000, Avg. Train Loss: 3.937484768629074\n",
      "Batch: 3000, Avg. Train Loss: 3.9459983892440795\n",
      "Batch: 4000, Avg. Train Loss: 3.947900619983673\n",
      "Batch: 5000, Avg. Train Loss: 3.9716800911426544\n",
      "Batch: 6000, Avg. Train Loss: 3.8900736733675\n",
      "Printing Validation Loss...\n",
      "Batch: 1000, Avg. Validation Loss: 3.399465307235718\n",
      "Epoch: 20\n",
      "Printing Train Loss...\n",
      "Batch: 1000, Avg. Train Loss: 3.929988430976868\n",
      "Batch: 2000, Avg. Train Loss: 3.9269162211418154\n",
      "Batch: 3000, Avg. Train Loss: 3.9284634993076324\n",
      "Batch: 4000, Avg. Train Loss: 3.915370374917984\n",
      "Batch: 5000, Avg. Train Loss: 3.888980260133743\n",
      "Batch: 6000, Avg. Train Loss: 3.924434308290482\n",
      "Printing Validation Loss...\n",
      "Batch: 1000, Avg. Validation Loss: 3.2948160082101823\n",
      "Model improved since last epoch! New Best Val Loss: 3.291917766721166\n",
      "Epoch: 21\n",
      "Printing Train Loss...\n",
      "Batch: 1000, Avg. Train Loss: 3.9535003414154053\n",
      "Batch: 2000, Avg. Train Loss: 3.899684281826019\n",
      "Batch: 3000, Avg. Train Loss: 3.8697341206073763\n",
      "Batch: 4000, Avg. Train Loss: 3.8232476187944413\n",
      "Batch: 5000, Avg. Train Loss: 3.9115903532505034\n",
      "Batch: 6000, Avg. Train Loss: 3.8995796840190886\n",
      "Printing Validation Loss...\n",
      "Batch: 1000, Avg. Validation Loss: 3.3517901792526246\n",
      "Epoch: 22\n",
      "Printing Train Loss...\n",
      "Batch: 1000, Avg. Train Loss: 3.8579559750556944\n",
      "Batch: 2000, Avg. Train Loss: 3.8947794535160063\n",
      "Batch: 3000, Avg. Train Loss: 3.860091389775276\n",
      "Batch: 4000, Avg. Train Loss: 3.9733105862140654\n",
      "Batch: 5000, Avg. Train Loss: 3.9370316848754885\n",
      "Batch: 6000, Avg. Train Loss: 3.90431337594986\n",
      "Printing Validation Loss...\n",
      "Batch: 1000, Avg. Validation Loss: 3.227286490917206\n",
      "Model improved since last epoch! New Best Val Loss: 3.249431776735691\n",
      "Epoch: 23\n",
      "Printing Train Loss...\n",
      "Batch: 1000, Avg. Train Loss: 3.812333105802536\n",
      "Batch: 2000, Avg. Train Loss: 3.8525016613006593\n",
      "Batch: 3000, Avg. Train Loss: 3.869125209093094\n",
      "Batch: 4000, Avg. Train Loss: 3.8582357164621355\n",
      "Batch: 5000, Avg. Train Loss: 3.879359622716904\n",
      "Batch: 6000, Avg. Train Loss: 3.8465782251358034\n",
      "Printing Validation Loss...\n",
      "Batch: 1000, Avg. Validation Loss: 3.3297989087104796\n",
      "Epoch: 24\n",
      "Printing Train Loss...\n",
      "Batch: 1000, Avg. Train Loss: 3.865546226143837\n",
      "Batch: 2000, Avg. Train Loss: 3.811852662563324\n",
      "Batch: 3000, Avg. Train Loss: 3.874571452617645\n",
      "Batch: 4000, Avg. Train Loss: 3.7856609315872194\n",
      "Batch: 5000, Avg. Train Loss: 3.8510212016105654\n",
      "Batch: 6000, Avg. Train Loss: 3.845429230451584\n",
      "Printing Validation Loss...\n",
      "Batch: 1000, Avg. Validation Loss: 3.172060727477074\n",
      "Model improved since last epoch! New Best Val Loss: 3.1814869347932992\n",
      "Epoch: 25\n",
      "Printing Train Loss...\n",
      "Batch: 1000, Avg. Train Loss: 3.840654716491699\n",
      "Batch: 2000, Avg. Train Loss: 3.807116521835327\n",
      "Batch: 3000, Avg. Train Loss: 3.8560899257659913\n",
      "Batch: 4000, Avg. Train Loss: 3.8735434448719026\n",
      "Batch: 5000, Avg. Train Loss: 3.779099568128586\n",
      "Batch: 6000, Avg. Train Loss: 3.76880607175827\n",
      "Printing Validation Loss...\n",
      "Batch: 1000, Avg. Validation Loss: 3.301638930797577\n",
      "Epoch: 26\n",
      "Printing Train Loss...\n",
      "Batch: 1000, Avg. Train Loss: 3.865784059047699\n",
      "Batch: 2000, Avg. Train Loss: 3.8539882793426514\n",
      "Batch: 3000, Avg. Train Loss: 3.769665840625763\n",
      "Batch: 4000, Avg. Train Loss: 3.8069350244998934\n",
      "Batch: 5000, Avg. Train Loss: 3.80015464091301\n",
      "Batch: 6000, Avg. Train Loss: 3.7748956978321075\n",
      "Printing Validation Loss...\n",
      "Batch: 1000, Avg. Validation Loss: 3.152023966908455\n",
      "Model improved since last epoch! New Best Val Loss: 3.16239535232372\n",
      "Epoch: 27\n",
      "Printing Train Loss...\n",
      "Batch: 1000, Avg. Train Loss: 3.7868817055225374\n",
      "Batch: 2000, Avg. Train Loss: 3.764159692764282\n",
      "Batch: 3000, Avg. Train Loss: 3.8177680759429933\n",
      "Batch: 4000, Avg. Train Loss: 3.7901023848056794\n",
      "Batch: 5000, Avg. Train Loss: 3.77871635556221\n",
      "Batch: 6000, Avg. Train Loss: 3.8181561906337738\n",
      "Printing Validation Loss...\n",
      "Batch: 1000, Avg. Validation Loss: 3.2641481690406797\n",
      "Epoch: 28\n",
      "Printing Train Loss...\n",
      "Batch: 1000, Avg. Train Loss: 3.7623281142711638\n",
      "Batch: 2000, Avg. Train Loss: 3.7892746138572693\n",
      "Batch: 3000, Avg. Train Loss: 3.7969984641075136\n",
      "Batch: 4000, Avg. Train Loss: 3.7807100987434388\n",
      "Batch: 5000, Avg. Train Loss: 3.7325902109146116\n",
      "Batch: 6000, Avg. Train Loss: 3.840970088481903\n",
      "Printing Validation Loss...\n",
      "Batch: 1000, Avg. Validation Loss: 3.1304847469329835\n",
      "Model improved since last epoch! New Best Val Loss: 3.142182780433371\n",
      "Epoch: 29\n",
      "Printing Train Loss...\n",
      "Batch: 1000, Avg. Train Loss: 3.775865667939186\n",
      "Batch: 2000, Avg. Train Loss: 3.7934819393157957\n",
      "Batch: 3000, Avg. Train Loss: 3.7660623633861543\n",
      "Batch: 4000, Avg. Train Loss: 3.774727260351181\n",
      "Batch: 5000, Avg. Train Loss: 3.713212894678116\n",
      "Batch: 6000, Avg. Train Loss: 3.751467706441879\n",
      "Printing Validation Loss...\n",
      "Batch: 1000, Avg. Validation Loss: 3.201422234296799\n",
      "Epoch: 30\n",
      "Printing Train Loss...\n",
      "Batch: 1000, Avg. Train Loss: 3.524725753545761\n",
      "Batch: 2000, Avg. Train Loss: 3.391248918056488\n",
      "Batch: 3000, Avg. Train Loss: 3.417880829334259\n",
      "Batch: 4000, Avg. Train Loss: 3.398199360251427\n",
      "Batch: 5000, Avg. Train Loss: 3.3579749232530594\n",
      "Batch: 6000, Avg. Train Loss: 3.332812122821808\n",
      "Printing Validation Loss...\n",
      "Batch: 1000, Avg. Validation Loss: 2.8043249384164812\n",
      "Model improved since last epoch! New Best Val Loss: 2.8170972805565127\n",
      "Epoch: 31\n",
      "Printing Train Loss...\n",
      "Batch: 1000, Avg. Train Loss: 3.2963755810260773\n",
      "Batch: 2000, Avg. Train Loss: 3.330770069241524\n",
      "Batch: 3000, Avg. Train Loss: 3.3130525394678116\n",
      "Batch: 4000, Avg. Train Loss: 3.314018345475197\n",
      "Batch: 5000, Avg. Train Loss: 3.277128568649292\n",
      "Batch: 6000, Avg. Train Loss: 3.3092622100114824\n",
      "Printing Validation Loss...\n",
      "Batch: 1000, Avg. Validation Loss: 2.7410745413303377\n",
      "Model improved since last epoch! New Best Val Loss: 2.7579534584054133\n",
      "Epoch: 32\n",
      "Printing Train Loss...\n",
      "Batch: 1000, Avg. Train Loss: 3.300636245846748\n",
      "Batch: 2000, Avg. Train Loss: 3.2820170477628707\n",
      "Batch: 3000, Avg. Train Loss: 3.2611018341779707\n",
      "Batch: 4000, Avg. Train Loss: 3.25584446310997\n",
      "Batch: 5000, Avg. Train Loss: 3.2443892766237257\n",
      "Batch: 6000, Avg. Train Loss: 3.2289496250152587\n",
      "Printing Validation Loss...\n",
      "Batch: 1000, Avg. Validation Loss: 2.7200253969430923\n",
      "Model improved since last epoch! New Best Val Loss: 2.7281512468177302\n",
      "Epoch: 33\n",
      "Printing Train Loss...\n",
      "Batch: 1000, Avg. Train Loss: 3.1916175414323806\n",
      "Batch: 2000, Avg. Train Loss: 3.2404118094444274\n",
      "Batch: 3000, Avg. Train Loss: 3.199145435094833\n",
      "Batch: 4000, Avg. Train Loss: 3.240399958252907\n",
      "Batch: 5000, Avg. Train Loss: 3.278496224522591\n",
      "Batch: 6000, Avg. Train Loss: 3.230897000193596\n",
      "Printing Validation Loss...\n",
      "Batch: 1000, Avg. Validation Loss: 2.711373281478882\n",
      "Model improved since last epoch! New Best Val Loss: 2.723424751800622\n",
      "Epoch: 34\n",
      "Printing Train Loss...\n",
      "Batch: 1000, Avg. Train Loss: 3.231737295150757\n",
      "Batch: 2000, Avg. Train Loss: 3.2417534630298617\n",
      "Batch: 3000, Avg. Train Loss: 3.188724569439888\n",
      "Batch: 4000, Avg. Train Loss: 3.24229549407959\n",
      "Batch: 5000, Avg. Train Loss: 3.203405549287796\n",
      "Batch: 6000, Avg. Train Loss: 3.2144001606702806\n",
      "Printing Validation Loss...\n",
      "Batch: 1000, Avg. Validation Loss: 2.7002428520917894\n",
      "Model improved since last epoch! New Best Val Loss: 2.7122072872969154\n",
      "Epoch: 35\n",
      "Printing Train Loss...\n",
      "Batch: 1000, Avg. Train Loss: 3.1827496322393416\n",
      "Batch: 2000, Avg. Train Loss: 3.218027646780014\n",
      "Batch: 3000, Avg. Train Loss: 3.1785081166028974\n",
      "Batch: 4000, Avg. Train Loss: 3.218319250702858\n",
      "Batch: 5000, Avg. Train Loss: 3.2336274762153625\n",
      "Batch: 6000, Avg. Train Loss: 3.189779280781746\n",
      "Printing Validation Loss...\n",
      "Batch: 1000, Avg. Validation Loss: 2.6654647179841997\n",
      "Model improved since last epoch! New Best Val Loss: 2.6737921818808585\n",
      "Epoch: 36\n",
      "Printing Train Loss...\n",
      "Batch: 1000, Avg. Train Loss: 3.1831000168323516\n",
      "Batch: 2000, Avg. Train Loss: 3.1665641741752624\n",
      "Batch: 3000, Avg. Train Loss: 3.2207683794498445\n",
      "Batch: 4000, Avg. Train Loss: 3.193721342086792\n",
      "Batch: 5000, Avg. Train Loss: 3.165084807872772\n",
      "Batch: 6000, Avg. Train Loss: 3.1484300236701968\n",
      "Printing Validation Loss...\n",
      "Batch: 1000, Avg. Validation Loss: 2.6700501730442046\n",
      "Epoch: 37\n",
      "Printing Train Loss...\n",
      "Batch: 1000, Avg. Train Loss: 3.191105159521103\n",
      "Batch: 2000, Avg. Train Loss: 3.1462054628133775\n",
      "Batch: 3000, Avg. Train Loss: 3.177157350420952\n",
      "Batch: 4000, Avg. Train Loss: 3.16812486743927\n",
      "Batch: 5000, Avg. Train Loss: 3.1676888217926025\n",
      "Batch: 6000, Avg. Train Loss: 3.201466930985451\n",
      "Printing Validation Loss...\n",
      "Batch: 1000, Avg. Validation Loss: 2.6581676697731016\n",
      "Model improved since last epoch! New Best Val Loss: 2.6661054696388917\n",
      "Epoch: 38\n",
      "Printing Train Loss...\n",
      "Batch: 1000, Avg. Train Loss: 3.2465889064073563\n",
      "Batch: 2000, Avg. Train Loss: 3.0833289296627044\n",
      "Batch: 3000, Avg. Train Loss: 3.182756868004799\n",
      "Batch: 4000, Avg. Train Loss: 3.1569989725351335\n",
      "Batch: 5000, Avg. Train Loss: 3.1612680481672286\n",
      "Batch: 6000, Avg. Train Loss: 3.135747844815254\n",
      "Printing Validation Loss...\n",
      "Batch: 1000, Avg. Validation Loss: 2.6269936709403994\n",
      "Model improved since last epoch! New Best Val Loss: 2.6370815255142053\n",
      "Epoch: 39\n",
      "Printing Train Loss...\n",
      "Batch: 1000, Avg. Train Loss: 3.1815230400562284\n",
      "Batch: 2000, Avg. Train Loss: 3.148449471235275\n",
      "Batch: 3000, Avg. Train Loss: 3.1264351279735565\n",
      "Batch: 4000, Avg. Train Loss: 3.1809432030916214\n",
      "Batch: 5000, Avg. Train Loss: 3.1506715207099916\n",
      "Batch: 6000, Avg. Train Loss: 3.1528411294221876\n",
      "Printing Validation Loss...\n",
      "Batch: 1000, Avg. Validation Loss: 2.63315317940712\n",
      "Epoch: 40\n",
      "Printing Train Loss...\n",
      "Batch: 1000, Avg. Train Loss: 3.1715266222953797\n",
      "Batch: 2000, Avg. Train Loss: 3.1285143065452576\n",
      "Batch: 3000, Avg. Train Loss: 3.1801713491678236\n",
      "Batch: 4000, Avg. Train Loss: 3.140572813510895\n",
      "Batch: 5000, Avg. Train Loss: 3.1175332568883896\n",
      "Batch: 6000, Avg. Train Loss: 3.109802087068558\n",
      "Printing Validation Loss...\n",
      "Batch: 1000, Avg. Validation Loss: 2.637688642501831\n",
      "Epoch: 41\n",
      "Printing Train Loss...\n",
      "Batch: 1000, Avg. Train Loss: 3.1863769714832304\n",
      "Batch: 2000, Avg. Train Loss: 3.073837179541588\n",
      "Batch: 3000, Avg. Train Loss: 3.121431856393814\n",
      "Batch: 4000, Avg. Train Loss: 3.1242965039014816\n",
      "Batch: 5000, Avg. Train Loss: 3.1544222024679183\n",
      "Batch: 6000, Avg. Train Loss: 3.1307329009771347\n",
      "Printing Validation Loss...\n",
      "Batch: 1000, Avg. Validation Loss: 2.6064430475234985\n",
      "Model improved since last epoch! New Best Val Loss: 2.6180863990883356\n",
      "Epoch: 42\n",
      "Printing Train Loss...\n",
      "Batch: 1000, Avg. Train Loss: 3.1102122082710264\n",
      "Batch: 2000, Avg. Train Loss: 3.1809893014431\n",
      "Batch: 3000, Avg. Train Loss: 3.1234455766677858\n",
      "Batch: 4000, Avg. Train Loss: 3.136485304951668\n",
      "Batch: 5000, Avg. Train Loss: 3.131238004684448\n",
      "Batch: 6000, Avg. Train Loss: 3.1125097444057466\n",
      "Printing Validation Loss...\n",
      "Batch: 1000, Avg. Validation Loss: 2.614613461613655\n",
      "Epoch: 43\n",
      "Printing Train Loss...\n",
      "Batch: 1000, Avg. Train Loss: 3.1046740822792054\n",
      "Batch: 2000, Avg. Train Loss: 3.1054897030591966\n",
      "Batch: 3000, Avg. Train Loss: 3.136441862463951\n",
      "Batch: 4000, Avg. Train Loss: 3.1645225834846498\n",
      "Batch: 5000, Avg. Train Loss: 3.0817651277780533\n",
      "Batch: 6000, Avg. Train Loss: 3.1170841344594957\n",
      "Printing Validation Loss...\n",
      "Batch: 1000, Avg. Validation Loss: 2.610538848280907\n",
      "Epoch: 44\n",
      "Printing Train Loss...\n",
      "Batch: 1000, Avg. Train Loss: 3.139827394723892\n",
      "Batch: 2000, Avg. Train Loss: 3.0745636088848114\n",
      "Batch: 3000, Avg. Train Loss: 3.11553306221962\n",
      "Batch: 4000, Avg. Train Loss: 3.1144774980545042\n",
      "Batch: 5000, Avg. Train Loss: 3.1369890213012694\n",
      "Batch: 6000, Avg. Train Loss: 3.1066963068246842\n",
      "Printing Validation Loss...\n",
      "Batch: 1000, Avg. Validation Loss: 2.603860710978508\n",
      "Model improved since last epoch! New Best Val Loss: 2.6120668130322735\n",
      "Epoch: 45\n",
      "Printing Train Loss...\n",
      "Batch: 1000, Avg. Train Loss: 3.0699294090270994\n",
      "Batch: 2000, Avg. Train Loss: 3.094446403503418\n",
      "Batch: 3000, Avg. Train Loss: 3.0876711477041243\n",
      "Batch: 4000, Avg. Train Loss: 3.0955963571071625\n",
      "Batch: 5000, Avg. Train Loss: 3.1077867950201035\n",
      "Batch: 6000, Avg. Train Loss: 3.1707753925323487\n",
      "Printing Validation Loss...\n",
      "Batch: 1000, Avg. Validation Loss: 2.6064108362197875\n",
      "Epoch: 46\n",
      "Printing Train Loss...\n",
      "Batch: 1000, Avg. Train Loss: 3.1150619806051254\n",
      "Batch: 2000, Avg. Train Loss: 3.11587411236763\n",
      "Batch: 3000, Avg. Train Loss: 3.144382038235664\n",
      "Batch: 4000, Avg. Train Loss: 3.121743533015251\n",
      "Batch: 5000, Avg. Train Loss: 3.115657935142517\n",
      "Batch: 6000, Avg. Train Loss: 3.042813355088234\n",
      "Printing Validation Loss...\n",
      "Batch: 1000, Avg. Validation Loss: 2.590291366100311\n",
      "Model improved since last epoch! New Best Val Loss: 2.6023085724676767\n",
      "Epoch: 47\n",
      "Printing Train Loss...\n",
      "Batch: 1000, Avg. Train Loss: 3.098521761059761\n",
      "Batch: 2000, Avg. Train Loss: 3.139755427122116\n",
      "Batch: 3000, Avg. Train Loss: 3.0949904245138167\n",
      "Batch: 4000, Avg. Train Loss: 3.079483835339546\n",
      "Batch: 5000, Avg. Train Loss: 3.1185605491399766\n",
      "Batch: 6000, Avg. Train Loss: 3.030173998236656\n",
      "Printing Validation Loss...\n",
      "Batch: 1000, Avg. Validation Loss: 2.59100726044178\n",
      "Model improved since last epoch! New Best Val Loss: 2.5999827988392226\n",
      "Epoch: 48\n",
      "Printing Train Loss...\n",
      "Batch: 1000, Avg. Train Loss: 3.1070296070575716\n",
      "Batch: 2000, Avg. Train Loss: 3.046964480638504\n",
      "Batch: 3000, Avg. Train Loss: 3.0995926996469496\n",
      "Batch: 4000, Avg. Train Loss: 3.1312108573913573\n",
      "Batch: 5000, Avg. Train Loss: 3.1060629106760027\n",
      "Batch: 6000, Avg. Train Loss: 3.078006752729416\n",
      "Printing Validation Loss...\n",
      "Batch: 1000, Avg. Validation Loss: 2.576416602253914\n",
      "Model improved since last epoch! New Best Val Loss: 2.5822542583451873\n",
      "Epoch: 49\n",
      "Printing Train Loss...\n",
      "Batch: 1000, Avg. Train Loss: 3.053831450343132\n",
      "Batch: 2000, Avg. Train Loss: 3.0809880154132845\n",
      "Batch: 3000, Avg. Train Loss: 3.1060901893377304\n",
      "Batch: 4000, Avg. Train Loss: 3.1119933098554613\n",
      "Batch: 5000, Avg. Train Loss: 3.0719153870344162\n",
      "Batch: 6000, Avg. Train Loss: 3.095447039961815\n",
      "Printing Validation Loss...\n",
      "Batch: 1000, Avg. Validation Loss: 2.5815140620470047\n",
      "Epoch: 50\n",
      "Printing Train Loss...\n",
      "Batch: 1000, Avg. Train Loss: 3.1056128031015398\n",
      "Batch: 2000, Avg. Train Loss: 3.014481023073196\n",
      "Batch: 3000, Avg. Train Loss: 3.068496555328369\n",
      "Batch: 4000, Avg. Train Loss: 3.086128605365753\n",
      "Batch: 5000, Avg. Train Loss: 3.1256034828424455\n",
      "Batch: 6000, Avg. Train Loss: 3.0903114203214646\n",
      "Printing Validation Loss...\n",
      "Batch: 1000, Avg. Validation Loss: 2.578360803484917\n",
      "Epoch: 51\n",
      "Printing Train Loss...\n",
      "Batch: 1000, Avg. Train Loss: 3.02448546397686\n",
      "Batch: 2000, Avg. Train Loss: 3.099841668844223\n",
      "Batch: 3000, Avg. Train Loss: 3.0402852293252947\n",
      "Batch: 4000, Avg. Train Loss: 3.1067338449954987\n",
      "Batch: 5000, Avg. Train Loss: 3.0634659930467607\n",
      "Batch: 6000, Avg. Train Loss: 3.0422374550104143\n",
      "Printing Validation Loss...\n",
      "Batch: 1000, Avg. Validation Loss: 2.5747466249465942\n",
      "Model improved since last epoch! New Best Val Loss: 2.58153909425343\n",
      "Epoch: 52\n",
      "Printing Train Loss...\n",
      "Batch: 1000, Avg. Train Loss: 3.085778464913368\n",
      "Batch: 2000, Avg. Train Loss: 3.073990026831627\n",
      "Batch: 3000, Avg. Train Loss: 3.053560309886932\n",
      "Batch: 4000, Avg. Train Loss: 3.052990633368492\n",
      "Batch: 5000, Avg. Train Loss: 3.074594774842262\n",
      "Batch: 6000, Avg. Train Loss: 3.0536713353395464\n",
      "Printing Validation Loss...\n",
      "Batch: 1000, Avg. Validation Loss: 2.549363512158394\n",
      "Model improved since last epoch! New Best Val Loss: 2.5579905933527134\n",
      "Epoch: 53\n",
      "Printing Train Loss...\n",
      "Batch: 1000, Avg. Train Loss: 3.04736405479908\n",
      "Batch: 2000, Avg. Train Loss: 3.076756404519081\n",
      "Batch: 3000, Avg. Train Loss: 3.0782349764108656\n",
      "Batch: 4000, Avg. Train Loss: 3.0872948596477507\n",
      "Batch: 5000, Avg. Train Loss: 3.056279223561287\n",
      "Batch: 6000, Avg. Train Loss: 3.056781182527542\n",
      "Printing Validation Loss...\n",
      "Batch: 1000, Avg. Validation Loss: 2.569027945756912\n",
      "Epoch: 54\n",
      "Printing Train Loss...\n",
      "Batch: 1000, Avg. Train Loss: 3.0448476401567457\n",
      "Batch: 2000, Avg. Train Loss: 3.020741998553276\n",
      "Batch: 3000, Avg. Train Loss: 3.048889880537987\n",
      "Batch: 4000, Avg. Train Loss: 3.0489578392505647\n",
      "Batch: 5000, Avg. Train Loss: 3.0877241723537443\n",
      "Batch: 6000, Avg. Train Loss: 3.1110831133127212\n",
      "Printing Validation Loss...\n",
      "Batch: 1000, Avg. Validation Loss: 2.574141630649567\n",
      "Epoch: 55\n",
      "Printing Train Loss...\n",
      "Batch: 1000, Avg. Train Loss: 3.070895489692688\n",
      "Batch: 2000, Avg. Train Loss: 3.003211322069168\n",
      "Batch: 3000, Avg. Train Loss: 3.079989809989929\n",
      "Batch: 4000, Avg. Train Loss: 3.0622019881010054\n",
      "Batch: 5000, Avg. Train Loss: 3.1495784349441527\n",
      "Batch: 6000, Avg. Train Loss: 3.0084423059225083\n",
      "Printing Validation Loss...\n",
      "Batch: 1000, Avg. Validation Loss: 2.5501130491495134\n",
      "Model improved since last epoch! New Best Val Loss: 2.5566307819411302\n",
      "Epoch: 56\n",
      "Printing Train Loss...\n",
      "Batch: 1000, Avg. Train Loss: 3.0783640896081925\n",
      "Batch: 2000, Avg. Train Loss: 3.036408217191696\n",
      "Batch: 3000, Avg. Train Loss: 3.0096554223299026\n",
      "Batch: 4000, Avg. Train Loss: 3.062365942955017\n",
      "Batch: 5000, Avg. Train Loss: 3.078772810935974\n",
      "Batch: 6000, Avg. Train Loss: 3.0582442611455916\n",
      "Printing Validation Loss...\n",
      "Batch: 1000, Avg. Validation Loss: 2.5432585102319716\n",
      "Model improved since last epoch! New Best Val Loss: 2.5558372012623614\n",
      "Epoch: 57\n",
      "Printing Train Loss...\n",
      "Batch: 1000, Avg. Train Loss: 3.0395536115169524\n",
      "Batch: 2000, Avg. Train Loss: 3.064726720929146\n",
      "Batch: 3000, Avg. Train Loss: 3.044379770755768\n",
      "Batch: 4000, Avg. Train Loss: 3.054669144034386\n",
      "Batch: 5000, Avg. Train Loss: 3.029814378142357\n",
      "Batch: 6000, Avg. Train Loss: 3.0232160485982895\n",
      "Printing Validation Loss...\n",
      "Batch: 1000, Avg. Validation Loss: 2.5596693633794785\n",
      "Epoch: 58\n",
      "Printing Train Loss...\n",
      "Batch: 1000, Avg. Train Loss: 3.0427185274362563\n",
      "Batch: 2000, Avg. Train Loss: 3.0453802585601806\n",
      "Batch: 3000, Avg. Train Loss: 3.01135249710083\n",
      "Batch: 4000, Avg. Train Loss: 2.9975981723070144\n",
      "Batch: 5000, Avg. Train Loss: 3.059527420401573\n",
      "Batch: 6000, Avg. Train Loss: 3.0423261939287185\n",
      "Printing Validation Loss...\n",
      "Batch: 1000, Avg. Validation Loss: 2.5476970728635786\n",
      "Epoch: 59\n",
      "Printing Train Loss...\n",
      "Batch: 1000, Avg. Train Loss: 3.024888190627098\n",
      "Batch: 2000, Avg. Train Loss: 3.0514841899871827\n",
      "Batch: 3000, Avg. Train Loss: 3.0325691144466402\n",
      "Batch: 4000, Avg. Train Loss: 3.043608173966408\n",
      "Batch: 5000, Avg. Train Loss: 3.0610015703439712\n",
      "Batch: 6000, Avg. Train Loss: 3.028829429149628\n",
      "Printing Validation Loss...\n",
      "Batch: 1000, Avg. Validation Loss: 2.545274110198021\n",
      "Model improved since last epoch! New Best Val Loss: 2.551557920790435\n",
      "Epoch: 60\n",
      "Printing Train Loss...\n",
      "Batch: 1000, Avg. Train Loss: 3.0213031835556032\n",
      "Batch: 2000, Avg. Train Loss: 2.990741923213005\n",
      "Batch: 3000, Avg. Train Loss: 3.0059804928302767\n",
      "Batch: 4000, Avg. Train Loss: 3.0212380591630934\n",
      "Batch: 5000, Avg. Train Loss: 2.9700460361242293\n",
      "Batch: 6000, Avg. Train Loss: 3.014320870757103\n",
      "Printing Validation Loss...\n",
      "Batch: 1000, Avg. Validation Loss: 2.4993851079940796\n",
      "Model improved since last epoch! New Best Val Loss: 2.5085180745570197\n",
      "Epoch: 61\n",
      "Printing Train Loss...\n",
      "Batch: 1000, Avg. Train Loss: 2.957188556075096\n",
      "Batch: 2000, Avg. Train Loss: 3.0142685651779173\n",
      "Batch: 3000, Avg. Train Loss: 2.9694496434926987\n",
      "Batch: 4000, Avg. Train Loss: 3.008762405157089\n",
      "Batch: 5000, Avg. Train Loss: 3.003590626001358\n",
      "Batch: 6000, Avg. Train Loss: 2.9997733731269838\n",
      "Printing Validation Loss...\n",
      "Batch: 1000, Avg. Validation Loss: 2.4964436693191527\n",
      "Model improved since last epoch! New Best Val Loss: 2.505803631381378\n",
      "Epoch: 62\n",
      "Printing Train Loss...\n",
      "Batch: 1000, Avg. Train Loss: 3.0180615483522417\n",
      "Batch: 2000, Avg. Train Loss: 2.985037747383118\n",
      "Batch: 3000, Avg. Train Loss: 2.9935617017745972\n",
      "Batch: 4000, Avg. Train Loss: 2.9579783519506453\n",
      "Batch: 5000, Avg. Train Loss: 2.982648042798042\n",
      "Batch: 6000, Avg. Train Loss: 2.955873205065727\n",
      "Printing Validation Loss...\n",
      "Batch: 1000, Avg. Validation Loss: 2.4954750549793245\n",
      "Model improved since last epoch! New Best Val Loss: 2.5047869657395636\n",
      "Epoch: 63\n",
      "Printing Train Loss...\n",
      "Batch: 1000, Avg. Train Loss: 2.9818964965343477\n",
      "Batch: 2000, Avg. Train Loss: 3.0292152775526047\n",
      "Batch: 3000, Avg. Train Loss: 2.978657692193985\n",
      "Batch: 4000, Avg. Train Loss: 2.9684160150289536\n",
      "Batch: 5000, Avg. Train Loss: 2.9360991697311403\n",
      "Batch: 6000, Avg. Train Loss: 2.98771595954895\n",
      "Printing Validation Loss...\n",
      "Batch: 1000, Avg. Validation Loss: 2.492330418586731\n",
      "Model improved since last epoch! New Best Val Loss: 2.502901703372335\n",
      "Epoch: 64\n",
      "Printing Train Loss...\n",
      "Batch: 1000, Avg. Train Loss: 2.9899060047864916\n",
      "Batch: 2000, Avg. Train Loss: 3.0216667637825014\n",
      "Batch: 3000, Avg. Train Loss: 2.9902787841558456\n",
      "Batch: 4000, Avg. Train Loss: 2.9690684517621992\n",
      "Batch: 5000, Avg. Train Loss: 2.9861490552425383\n",
      "Batch: 6000, Avg. Train Loss: 2.9521764940023423\n",
      "Printing Validation Loss...\n",
      "Batch: 1000, Avg. Validation Loss: 2.485710496306419\n",
      "Model improved since last epoch! New Best Val Loss: 2.4959239865501748\n",
      "Epoch: 65\n",
      "Printing Train Loss...\n",
      "Batch: 1000, Avg. Train Loss: 2.970383748292923\n",
      "Batch: 2000, Avg. Train Loss: 2.988650046467781\n",
      "Batch: 3000, Avg. Train Loss: 2.9791110937595366\n",
      "Batch: 4000, Avg. Train Loss: 2.9528195732831954\n",
      "Batch: 5000, Avg. Train Loss: 2.939730175971985\n",
      "Batch: 6000, Avg. Train Loss: 3.00671528172493\n",
      "Printing Validation Loss...\n",
      "Batch: 1000, Avg. Validation Loss: 2.4871029062271117\n",
      "Epoch: 66\n",
      "Printing Train Loss...\n",
      "Batch: 1000, Avg. Train Loss: 2.998510533571243\n",
      "Batch: 2000, Avg. Train Loss: 2.9290724691152574\n",
      "Batch: 3000, Avg. Train Loss: 2.9473304131031037\n",
      "Batch: 4000, Avg. Train Loss: 3.0128699620962145\n",
      "Batch: 5000, Avg. Train Loss: 2.990285844087601\n",
      "Batch: 6000, Avg. Train Loss: 2.971496882915497\n",
      "Printing Validation Loss...\n",
      "Batch: 1000, Avg. Validation Loss: 2.480079183340073\n",
      "Model improved since last epoch! New Best Val Loss: 2.491068203647023\n",
      "Epoch: 67\n",
      "Printing Train Loss...\n",
      "Batch: 1000, Avg. Train Loss: 2.961188675880432\n",
      "Batch: 2000, Avg. Train Loss: 2.9889702084064482\n",
      "Batch: 3000, Avg. Train Loss: 3.005557976484299\n",
      "Batch: 4000, Avg. Train Loss: 2.9483174529075624\n",
      "Batch: 5000, Avg. Train Loss: 2.976026382088661\n",
      "Batch: 6000, Avg. Train Loss: 2.965914359688759\n",
      "Printing Validation Loss...\n",
      "Batch: 1000, Avg. Validation Loss: 2.483538106918335\n",
      "Epoch: 68\n",
      "Printing Train Loss...\n",
      "Batch: 1000, Avg. Train Loss: 2.952263377547264\n",
      "Batch: 2000, Avg. Train Loss: 2.9395747917890547\n",
      "Batch: 3000, Avg. Train Loss: 3.0022723821401596\n",
      "Batch: 4000, Avg. Train Loss: 3.0073767684698103\n",
      "Batch: 5000, Avg. Train Loss: 2.971621949672699\n",
      "Batch: 6000, Avg. Train Loss: 2.976958437681198\n",
      "Printing Validation Loss...\n",
      "Batch: 1000, Avg. Validation Loss: 2.4867974799871444\n",
      "Epoch: 69\n",
      "Printing Train Loss...\n",
      "Batch: 1000, Avg. Train Loss: 2.920669450044632\n",
      "Batch: 2000, Avg. Train Loss: 3.000968438744545\n",
      "Batch: 3000, Avg. Train Loss: 2.915561584711075\n",
      "Batch: 4000, Avg. Train Loss: 2.9713345552682875\n",
      "Batch: 5000, Avg. Train Loss: 3.009587828040123\n",
      "Batch: 6000, Avg. Train Loss: 2.9875585678815844\n",
      "Printing Validation Loss...\n",
      "Batch: 1000, Avg. Validation Loss: 2.4809574226140976\n",
      "Model improved since last epoch! New Best Val Loss: 2.490849449851013\n",
      "Epoch: 70\n",
      "Printing Train Loss...\n",
      "Batch: 1000, Avg. Train Loss: 2.9800826070308686\n",
      "Batch: 2000, Avg. Train Loss: 3.0237690627574922\n",
      "Batch: 3000, Avg. Train Loss: 2.966747274518013\n",
      "Batch: 4000, Avg. Train Loss: 2.934493642807007\n",
      "Batch: 5000, Avg. Train Loss: 2.9479025291204453\n",
      "Batch: 6000, Avg. Train Loss: 2.967940069198608\n",
      "Printing Validation Loss...\n",
      "Batch: 1000, Avg. Validation Loss: 2.4868416436910628\n",
      "Epoch: 71\n",
      "Printing Train Loss...\n",
      "Batch: 1000, Avg. Train Loss: 2.9785783059597017\n",
      "Batch: 2000, Avg. Train Loss: 2.9492522559165955\n",
      "Batch: 3000, Avg. Train Loss: 2.9567313191890716\n",
      "Batch: 4000, Avg. Train Loss: 2.948696823596954\n",
      "Batch: 5000, Avg. Train Loss: 2.974825900673866\n",
      "Batch: 6000, Avg. Train Loss: 2.9255467674732207\n",
      "Printing Validation Loss...\n",
      "Batch: 1000, Avg. Validation Loss: 2.478845412015915\n",
      "Model improved since last epoch! New Best Val Loss: 2.4884049714117715\n",
      "Epoch: 72\n",
      "Printing Train Loss...\n",
      "Batch: 1000, Avg. Train Loss: 3.002621949791908\n",
      "Batch: 2000, Avg. Train Loss: 2.973538203597069\n",
      "Batch: 3000, Avg. Train Loss: 2.9325991557836533\n",
      "Batch: 4000, Avg. Train Loss: 2.9658511669635774\n",
      "Batch: 5000, Avg. Train Loss: 2.960036244273186\n",
      "Batch: 6000, Avg. Train Loss: 2.962309252023697\n",
      "Printing Validation Loss...\n",
      "Batch: 1000, Avg. Validation Loss: 2.474340708494186\n",
      "Model improved since last epoch! New Best Val Loss: 2.484991882066957\n",
      "Epoch: 73\n",
      "Printing Train Loss...\n",
      "Batch: 1000, Avg. Train Loss: 2.9529658241271974\n",
      "Batch: 2000, Avg. Train Loss: 3.01422488617897\n",
      "Batch: 3000, Avg. Train Loss: 2.976210584163666\n",
      "Batch: 4000, Avg. Train Loss: 2.9597906593084335\n",
      "Batch: 5000, Avg. Train Loss: 2.9430066373348236\n",
      "Batch: 6000, Avg. Train Loss: 2.9401478397846224\n",
      "Printing Validation Loss...\n",
      "Batch: 1000, Avg. Validation Loss: 2.47181772005558\n",
      "Model improved since last epoch! New Best Val Loss: 2.481508626726077\n",
      "Epoch: 74\n",
      "Printing Train Loss...\n",
      "Batch: 1000, Avg. Train Loss: 2.995987444281578\n",
      "Batch: 2000, Avg. Train Loss: 2.9239297161102296\n",
      "Batch: 3000, Avg. Train Loss: 2.977036501288414\n",
      "Batch: 4000, Avg. Train Loss: 2.958501801133156\n",
      "Batch: 5000, Avg. Train Loss: 2.998314751505852\n",
      "Batch: 6000, Avg. Train Loss: 2.95190900015831\n",
      "Printing Validation Loss...\n",
      "Batch: 1000, Avg. Validation Loss: 2.474255601644516\n",
      "Epoch: 75\n",
      "Printing Train Loss...\n",
      "Batch: 1000, Avg. Train Loss: 2.963091420531273\n",
      "Batch: 2000, Avg. Train Loss: 2.9800071300268174\n",
      "Batch: 3000, Avg. Train Loss: 2.9752315123081208\n",
      "Batch: 4000, Avg. Train Loss: 2.968185463190079\n",
      "Batch: 5000, Avg. Train Loss: 2.948814497947693\n",
      "Batch: 6000, Avg. Train Loss: 2.968622395157814\n",
      "Printing Validation Loss...\n",
      "Batch: 1000, Avg. Validation Loss: 2.477611347436905\n",
      "Epoch: 76\n",
      "Printing Train Loss...\n",
      "Batch: 1000, Avg. Train Loss: 2.954267918229103\n",
      "Batch: 2000, Avg. Train Loss: 2.958634637951851\n",
      "Batch: 3000, Avg. Train Loss: 2.9985342725515367\n",
      "Batch: 4000, Avg. Train Loss: 2.9467635411024093\n",
      "Batch: 5000, Avg. Train Loss: 2.919413115978241\n",
      "Batch: 6000, Avg. Train Loss: 2.975289325118065\n",
      "Printing Validation Loss...\n",
      "Batch: 1000, Avg. Validation Loss: 2.4761663377285004\n",
      "Epoch: 77\n",
      "Printing Train Loss...\n",
      "Batch: 1000, Avg. Train Loss: 2.970633642077446\n",
      "Batch: 2000, Avg. Train Loss: 2.9430541162490846\n",
      "Batch: 3000, Avg. Train Loss: 2.965633414745331\n",
      "Batch: 4000, Avg. Train Loss: 2.987760571837425\n",
      "Batch: 5000, Avg. Train Loss: 2.8853949631452562\n",
      "Batch: 6000, Avg. Train Loss: 3.009453342795372\n",
      "Printing Validation Loss...\n",
      "Batch: 1000, Avg. Validation Loss: 2.477609032154083\n",
      "Epoch: 78\n",
      "Printing Train Loss...\n",
      "Batch: 1000, Avg. Train Loss: 2.9036423008441927\n",
      "Batch: 2000, Avg. Train Loss: 3.0024956773519516\n",
      "Batch: 3000, Avg. Train Loss: 2.975202614426613\n",
      "Batch: 4000, Avg. Train Loss: 2.9612315291166307\n",
      "Batch: 5000, Avg. Train Loss: 2.993592847585678\n",
      "Batch: 6000, Avg. Train Loss: 2.976281750679016\n",
      "Printing Validation Loss...\n",
      "Batch: 1000, Avg. Validation Loss: 2.474565465331078\n",
      "Epoch: 79\n",
      "Printing Train Loss...\n",
      "Batch: 1000, Avg. Train Loss: 3.0052063403129576\n",
      "Batch: 2000, Avg. Train Loss: 2.9816850187778474\n",
      "Batch: 3000, Avg. Train Loss: 2.9169531183242796\n",
      "Batch: 4000, Avg. Train Loss: 2.9858070253133775\n",
      "Batch: 5000, Avg. Train Loss: 2.957706931114197\n",
      "Batch: 6000, Avg. Train Loss: 2.906725304603577\n",
      "Printing Validation Loss...\n",
      "Batch: 1000, Avg. Validation Loss: 2.470249482035637\n",
      "Model improved since last epoch! New Best Val Loss: 2.4792288825217605\n",
      "Epoch: 80\n",
      "Printing Train Loss...\n",
      "Batch: 1000, Avg. Train Loss: 2.934345946550369\n",
      "Batch: 2000, Avg. Train Loss: 2.971964335680008\n",
      "Batch: 3000, Avg. Train Loss: 2.995069383740425\n",
      "Batch: 4000, Avg. Train Loss: 2.957347377896309\n",
      "Batch: 5000, Avg. Train Loss: 2.9559830844402315\n",
      "Batch: 6000, Avg. Train Loss: 2.9773135150671006\n",
      "Printing Validation Loss...\n",
      "Batch: 1000, Avg. Validation Loss: 2.4758131206035614\n",
      "Epoch: 81\n",
      "Printing Train Loss...\n",
      "Batch: 1000, Avg. Train Loss: 2.983926506757736\n",
      "Batch: 2000, Avg. Train Loss: 2.96403926718235\n",
      "Batch: 3000, Avg. Train Loss: 2.9706366533041\n",
      "Batch: 4000, Avg. Train Loss: 2.9662172434329985\n",
      "Batch: 5000, Avg. Train Loss: 2.9305620012283327\n",
      "Batch: 6000, Avg. Train Loss: 2.9426527997255327\n",
      "Printing Validation Loss...\n",
      "Batch: 1000, Avg. Validation Loss: 2.473556861400604\n",
      "Epoch: 82\n",
      "Printing Train Loss...\n",
      "Batch: 1000, Avg. Train Loss: 2.9717460880279543\n",
      "Batch: 2000, Avg. Train Loss: 2.926689598798752\n",
      "Batch: 3000, Avg. Train Loss: 2.9250727384090425\n",
      "Batch: 4000, Avg. Train Loss: 2.95438300716877\n",
      "Batch: 5000, Avg. Train Loss: 2.9281421819925306\n",
      "Batch: 6000, Avg. Train Loss: 3.012586124420166\n",
      "Printing Validation Loss...\n",
      "Batch: 1000, Avg. Validation Loss: 2.466380975008011\n",
      "Model improved since last epoch! New Best Val Loss: 2.475598941434839\n",
      "Epoch: 83\n",
      "Printing Train Loss...\n",
      "Batch: 1000, Avg. Train Loss: 2.9460913578271866\n",
      "Batch: 2000, Avg. Train Loss: 2.990182970523834\n",
      "Batch: 3000, Avg. Train Loss: 2.9426428334712984\n",
      "Batch: 4000, Avg. Train Loss: 2.9697162284851073\n",
      "Batch: 5000, Avg. Train Loss: 2.954284823536873\n",
      "Batch: 6000, Avg. Train Loss: 2.9443924516439437\n",
      "Printing Validation Loss...\n",
      "Batch: 1000, Avg. Validation Loss: 2.4701290100812914\n",
      "Epoch: 84\n",
      "Printing Train Loss...\n",
      "Batch: 1000, Avg. Train Loss: 2.945205293416977\n",
      "Batch: 2000, Avg. Train Loss: 2.9015804070234297\n",
      "Batch: 3000, Avg. Train Loss: 2.9568841370344163\n",
      "Batch: 4000, Avg. Train Loss: 2.9733843184709547\n",
      "Batch: 5000, Avg. Train Loss: 2.9525323214530945\n",
      "Batch: 6000, Avg. Train Loss: 2.959165190577507\n",
      "Printing Validation Loss...\n",
      "Batch: 1000, Avg. Validation Loss: 2.470207999229431\n",
      "Epoch: 85\n",
      "Printing Train Loss...\n",
      "Batch: 1000, Avg. Train Loss: 2.9429449603557587\n",
      "Batch: 2000, Avg. Train Loss: 2.932891910791397\n",
      "Batch: 3000, Avg. Train Loss: 2.9979022661447523\n",
      "Batch: 4000, Avg. Train Loss: 2.9466831078529356\n",
      "Batch: 5000, Avg. Train Loss: 2.9505533846616747\n",
      "Batch: 6000, Avg. Train Loss: 2.9517798255681993\n",
      "Printing Validation Loss...\n",
      "Batch: 1000, Avg. Validation Loss: 2.4676908967494966\n",
      "Epoch: 86\n",
      "Printing Train Loss...\n",
      "Batch: 1000, Avg. Train Loss: 2.9375431176424027\n",
      "Batch: 2000, Avg. Train Loss: 2.954540076494217\n",
      "Batch: 3000, Avg. Train Loss: 2.929581236720085\n",
      "Batch: 4000, Avg. Train Loss: 2.9452907497882843\n",
      "Batch: 5000, Avg. Train Loss: 2.9828395065069198\n",
      "Batch: 6000, Avg. Train Loss: 2.9501491665840147\n",
      "Printing Validation Loss...\n",
      "Batch: 1000, Avg. Validation Loss: 2.4727631704807282\n",
      "Epoch: 87\n",
      "Printing Train Loss...\n",
      "Batch: 1000, Avg. Train Loss: 2.9595714745521544\n",
      "Batch: 2000, Avg. Train Loss: 2.9148341386318206\n",
      "Batch: 3000, Avg. Train Loss: 2.9373248093128206\n",
      "Batch: 4000, Avg. Train Loss: 2.9830200538635254\n",
      "Batch: 5000, Avg. Train Loss: 2.954031120181084\n",
      "Batch: 6000, Avg. Train Loss: 2.9183913905620575\n",
      "Printing Validation Loss...\n",
      "Batch: 1000, Avg. Validation Loss: 2.472728575229645\n",
      "Epoch: 88\n",
      "Printing Train Loss...\n",
      "Batch: 1000, Avg. Train Loss: 2.9817840859889984\n",
      "Batch: 2000, Avg. Train Loss: 2.930076592564583\n",
      "Batch: 3000, Avg. Train Loss: 2.9541165276765824\n",
      "Batch: 4000, Avg. Train Loss: 2.963814088344574\n",
      "Batch: 5000, Avg. Train Loss: 2.9477931524515153\n",
      "Batch: 6000, Avg. Train Loss: 2.9357272973060606\n",
      "Printing Validation Loss...\n",
      "Batch: 1000, Avg. Validation Loss: 2.473271607875824\n",
      "Epoch: 89\n",
      "Printing Train Loss...\n",
      "Batch: 1000, Avg. Train Loss: 2.926147516012192\n",
      "Batch: 2000, Avg. Train Loss: 2.9353021216392516\n",
      "Batch: 3000, Avg. Train Loss: 2.9626436408758163\n",
      "Batch: 4000, Avg. Train Loss: 2.943519110918045\n",
      "Batch: 5000, Avg. Train Loss: 2.942602037668228\n",
      "Batch: 6000, Avg. Train Loss: 2.9908049712181093\n",
      "Printing Validation Loss...\n",
      "Batch: 1000, Avg. Validation Loss: 2.4646216210126877\n",
      "Model improved since last epoch! New Best Val Loss: 2.474209497755139\n",
      "Finished Training\n"
     ]
    }
   ],
   "source": [
    "try:\n",
    "    # train your network\n",
    "    n_epochs = 90 # start small, and increase when you've decided on your model structure and hyperparams\n",
    "    train_net(n_epochs)\n",
    "except KeyboardInterrupt:\n",
    "    print('Stopping Training...')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**12. Plot Training Loss History:-**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<matplotlib.legend.Legend at 0x7f1de8df94d0>"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAWoAAAEICAYAAAB25L6yAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nO3deXxV1d3v8c8vCSQMYQ6KBBlEcGAIGEVFEYXaigjWoc5C7VOvto9DrdW2L60+Wmrb63NreV2t1zq11kfrrHWsWhVbqzKICgIiCDLKJDOBDOv+8TuHk8SQHEJOzj453/frtV8nOXtaZ4vfs7L22mtZCAEREYmunHQXQERE6qegFhGJOAW1iEjEKahFRCJOQS0iEnEKahGRiFNQi0SAmY02s+XpLodEk4JaGmRmS8xsbLrLIZKtFNSS1cwsL91lEGmIgloazczyzewOM1sZW+4ws/zYum5m9ryZbTSzDWb2tpnlxNZdb2YrzGyLmS0wszF7OH5HM/uzma01s6VmdoOZ5cTOu9HMBlXbtsjMdphZ99jv481sdmy7d8xsSLVtl8TK8BGwra6wNrNDzOzVWNkXmNl3qq170Mzujq3fYmZvmVnvauuPNbPpZrYp9npstXVdzOyB2PX6ysyeqXXeH5vZGjNbZWbfrfb+ODP7JHa+FWZ27V79x5LMFkLQoqXeBVgCjK3j/VuAd4HuQBHwDnBrbN1twN1Aq9hyPGDAQGAZcEBsuz7AQXs475+BZ4HC2HafAt+LrbsfmFJt2x8CL8d+Hg6sAUYAucCk2GfIr/Z5ZgO9gDZ1nLddrIzfBfJix1sHHB5b/yCwBRgF5AO/B/4ZW9cF+Aq4KLbvebHfu8bWvwD8Fegcuy4nxN4fDVTErmkrYBywHegcW78KOD72c2dgeLr/XWhpviXtBdAS/aWeoF4EjKv2+zeBJbGfb4mFbP9a+/SPhehYoFU958wFdgKHVXvvfwFvxn4eCyyutu5fwMWxn/8Q/8Kotn5BtVBcAlxSz7nPAd6u9d7/A26K/fwg8Gi1de2ByljwXwS8X2vffwOTgR5AVTx8a20zGtgB5FV7bw1wdOznL2Kfv0O6/z1oaf5FTR+yLw4Allb7fWnsPYD/DXwG/N3MFpvZTwFCCJ8BVwM3A2vM7FEzO4Cv6wa0ruP4PWM//wNoY2YjYs0OJcDTsXW9gR/Hmj02mtlGPESrn2dZPZ+rNzCi1v4XAPvXtX8IYSuwIXb82tekerl7ARtCCF/t4bzrQwgV1X7fjn8JAJyJ17KXxppajqmn/NLCKKhlX6zEQy3uwNh7hBC2hBB+HELoB5wGXBNviw4h/E8I4bjYvgH4TR3HXgeU13H8FbFjVAGP4U0L5wPPhxC2xLZbhjeLdKq2tA0hPFLtWPUNG7kMeKvW/u1DCJdX26ZX/Acza483eays45pUL/cyoIuZdarn3HUKIUwPIUzEm5meiX12yRIKaklWKzMrqLbkAY8AN8Ru5HUDfgH8BXbfzOtvZgZsxpsGKs1soJmdFLvpWIb/uV9Z+2QhhEo8jKaYWWGs1nxN/Pgx/4M3U1wQ+znuj8Blsdq2mVk7MzvVzAqT/KzPAwPM7CIzaxVbjjSzQ6ttM87MjjOz1sCtwHshhGXAi7F9zzezPDM7BzgM/yJZBbwE3GVmnWPHHdVQYcystZldYGYdQwjlJK6nZIt0t71oif6Ct+mGWssvgQJgKn6ja1Xs54LYPj+K7bcNWA7cGHt/CPA+fjNuAx6KB+zhvJ3xYF6L10Z/AeTU2uaz2HFa13r/W8B0YGOsbI8DhdU+z9fa3GvtPxC/8bcWWI83tZTE1j2I3yh9FdgKTAP6Vtv3OGAmsCn2ely1dV2APwFf4jcZn4q9PxpYXsd1H4s3Ab0c235z7HMdV1/5tbSsxWL/IEQkSWb2IB6qN6S7LJId1PQhIhJxCmoRkYhT04eISMSpRi0iEnEpGZCmW7duoU+fPqk4tIhIizRz5sx1IYSiutalJKj79OnDjBkzUnFoEZEWycxqP9G6m5o+REQiTkEtIhJxCmoRkYjT7BYiGay8vJzly5dTVlaW7qJIkgoKCiguLqZVq1ZJ76OgFslgy5cvp7CwkD59+uDjX0mUhRBYv349y5cvp2/fvknvp6YPkQxWVlZG165dFdIZwszo2rXrXv8FpKAWyXAK6czSmP9ekQrqW2+FV15JdylERKIlUkH9298qqEUyyfr16ykpKaGkpIT999+fnj177v59165d9e47Y8YMrrzyyr06X58+fVi3bt2+FDkjRepmYmEhbN2a7lKISLK6du3K7NmzAbj55ptp374911577e71FRUV5OXVHTOlpaWUlpY2SzkzXaRq1O3bw5YtDW8nItE1efJkrrnmGk488USuv/563n//fY499liGDRvGsccey4IFCwB48803GT9+POAhf8kllzB69Gj69evH1KlTkz7f0qVLGTNmDEOGDGHMmDF88cUXADz++OMMGjSIoUOHMmqUz3g2d+5cjjrqKEpKShgyZAgLFy5s4k+fGqpRi7QQV18NscptkykpgTvu2Pv9Pv30U1577TVyc3PZvHkz06ZNIy8vj9dee42f//znPPnkk1/bZ/78+bzxxhts2bKFgQMHcvnllyfV1/g///M/ufjii5k0aRL3338/V155Jc888wy33HILr7zyCj179mTjxo0A3H333Vx11VVccMEF7Nq1i8rKzJh6MlJBrRq1SMtw9tlnk5ubC8CmTZuYNGkSCxcuxMwoLy+vc59TTz2V/Px88vPz6d69O19++SXFxcUNnuvf//43Tz31FAAXXXQR1113HQAjR45k8uTJfOc73+GMM84A4JhjjmHKlCksX76cM844g4MPPrgpPm7KRSqoCwth9ep0l0IkMzWm5psq7dq12/3zjTfeyIknnsjTTz/NkiVLGD16dJ375Ofn7/45NzeXioqKRp073v3t7rvv5r333uOFF16gpKSE2bNnc/755zNixAheeOEFvvnNb3Lvvfdy0kknNeo8zSmpNmozu8rM5pjZXDO7OlWFUY1apOXZtGkTPXv2BODBBx9s8uMfe+yxPProowA8/PDDHHfccQAsWrSIESNGcMstt9CtWzeWLVvG4sWL6devH1deeSUTJkzgo48+avLypEKDQW1mg4DvA0cBQ4HxZpaSvxfURi3S8lx33XX87Gc/Y+TIkU3SJjxkyBCKi4spLi7mmmuuYerUqTzwwAMMGTKEhx56iN///vcA/OQnP2Hw4MEMGjSIUaNGMXToUP76178yaNAgSkpKmD9/PhdffPE+l6c5NDhnopmdDXwzhPAfsd9vBHaGEH67p31KS0tDYyYO+NGP4L77YPPmvd5VJCvNmzePQw89NN3FkL1U1383M5sZQqizv2IyTR9zgFFm1tXM2gLjgF61NzKzS81shpnNWLt2bSOKnqhRa75dEZGEBoM6hDAP+A3wKvAy8CHwtVb+EMI9IYTSEEJpUVGd0341qH17D+nt2xu1u4hIi5TUzcQQwn0hhOEhhFHABiAlvcQLC/1V7dQiIgnJ9vroHns9EDgDeCQVhWnf3l/V80NEJCHZftRPmllXoBz4YQjhq1QURjVqEZGvSyqoQwjHp7ogoBq1iEhdIjUok2rUIpll9OjRvFJrbOI77riDH/zgB/XuE+++O27cuN3jcFR38803c/vtt9d77meeeYZPPvlk9++/+MUveO211/am+HWqPlhUVEQqqFWjFsks55133u6nAuMeffRRzjvvvKT2f/HFF+nUqVOjzl07qG+55RbGjh3bqGNFXSSDWjVqkcxw1lln8fzzz7Nz504AlixZwsqVKznuuOO4/PLLKS0t5fDDD+emm26qc//qEwFMmTKFgQMHMnbs2N1DoQL88Y9/5Mgjj2To0KGceeaZbN++nXfeeYfnnnuOn/zkJ5SUlLBo0SImT57ME088AcDrr7/OsGHDGDx4MJdccsnu8vXp04ebbrqJ4cOHM3jwYObPn5/0Z33kkUd2P+l4/fXXA1BZWcnkyZMZNGgQgwcP5ne/+x0AU6dO5bDDDmPIkCGce+65e3lVvy5ygzKBglqkUdIwzmnXrl056qijePnll5k4cSKPPvoo55xzDmbGlClT6NKlC5WVlYwZM4aPPvqIIUOG1HmcmTNn8uijj/LBBx9QUVHB8OHDOeKIIwA444wz+P73vw/ADTfcwH333ccVV1zBhAkTGD9+PGeddVaNY5WVlTF58mRef/11BgwYwMUXX8wf/vAHrr7ahynq1q0bs2bN4q677uL222/n3nvvbfAyrFy5kuuvv56ZM2fSuXNnTj75ZJ555hl69erFihUrmDNnDsDuZpxf//rXfP755+Tn59fZtLO3IlmjVtOHSOao3vxRvdnjscceY/jw4QwbNoy5c+fWaKao7e233+bb3/42bdu2pUOHDkyYMGH3ujlz5nD88cczePBgHn74YebOnVtveRYsWEDfvn0ZMGAAAJMmTWLatGm718eHPD3iiCNYsmRJUp9x+vTpjB49mqKiIvLy8rjggguYNm0a/fr1Y/HixVxxxRW8/PLLdOjQAfDxSC644AL+8pe/7HGGm70RqRp169a+qEYt0ghpGuf09NNP55prrmHWrFns2LGD4cOH8/nnn3P77bczffp0OnfuzOTJkykrK6v3OHuanXvy5Mk888wzDB06lAcffJA333yz3uM0NH5RfDjVvRlKdU/H7Ny5Mx9++CGvvPIKd955J4899hj3338/L7zwAtOmTeO5557j1ltvZe7cufsU2JGqUYOGOhXJNO3bt2f06NFccsklu2vTmzdvpl27dnTs2JEvv/ySl156qd5jjBo1iqeffpodO3awZcsW/va3v+1et2XLFnr06EF5eTkPP/zw7vcLCwvZUkdYHHLIISxZsoTPPvsMgIceeogTTjhhnz7jiBEjeOutt1i3bh2VlZU88sgjnHDCCaxbt46qqirOPPNMbr31VmbNmkVVVRXLli3jxBNP5Le//S0bN25k6z7WPiNVowYNdSqSic477zzOOOOM3U0gQ4cOZdiwYRx++OH069ePkSNH1rv/8OHDOeeccygpKaF3794cf3zi0Y1bb72VESNG0Lt3bwYPHrw7nM8991y+//3vM3Xq1N03EQEKCgp44IEHOPvss6moqODII4/ksssu26vP8/rrr9eYXebxxx/ntttu48QTTySEwLhx45g4cSIffvgh3/3ud6mqqgLgtttuo7KykgsvvJBNmzYRQuBHP/pRo3u2xDU4zGljNHaYU4BBg2DgQKhjSjURqUXDnGamVAxz2qxUoxYRqSlyQa02ahGRmiIX1KpRi+ydVDRfSuo05r9X5IJaNWqR5BUUFLB+/XqFdYYIIbB+/XoKCgr2aj/1+hDJYMXFxSxfvpzGTn8nza+goKBGj5JkRC6oVaMWSV6rVq3o27dvuoshKRa5po/CQti5E8rL010SEZFoiFxQawQ9EZGaIhfUGkFPRKSmyAW1RtATEakpckGtGrWISE1JBbWZ/cjM5prZHDN7xMz2rhPgXlCNWkSkpgaD2sx6AlcCpSGEQUAusO9zy+yBatQiIjUl2/SRB7QxszygLbAyVQVSjVpEpKYGgzqEsAK4HfgCWAVsCiH8vfZ2Znapmc0wsxn78pSUatQiIjUl0/TRGZgI9AUOANqZ2YW1twsh3BNCKA0hlBYVFTW6QKpRi4jUlEzTx1jg8xDC2hBCOfAUcGyqCtS2LZipRi0iEpdMUH8BHG1mbc1nnxwDzEtZgXKgXTvVqEVE4pJpo34PeAKYBXwc2+eeVBaqfXvVqEVE4pIaPS+EcBNwU4rLslthoWrUIiJxkXsyEVSjFhGpLpJBrRq1iEhCJINaNWoRkYRIBrVq1CIiCZEMatWoRUQSIhnUqlGLiCREMqjjNeoQ0l0SEZH0i2RQFxZ6SO/Yke6SiIikXySDWgMziYgkRDKoNdSpiEhCJINaNWoRkYRIBrVq1CIiCZEMatWoRUQSIhnUqlGLiCREMqhVoxYRSYhkUKtGLSKSEMmgVo1aRCQhkkHdurUvqlGLiEQ0qMFr1apRi4hEOKgLC1WjFhGBJILazAaa2exqy2YzuzrVBVONWkTENTgLeQhhAVACYGa5wArg6RSXS5MHiIjE7G3TxxhgUQhhaSoKU50mDxARcXsb1OcCj9S1wswuNbMZZjZj7dq1+1ww1ahFRFzSQW1mrYEJwON1rQ8h3BNCKA0hlBYVFe1zwVSjFhFxe1OjPgWYFUL4MlWFqU41ahERtzdBfR57aPZIhU6dYONGqKxsrjOKiERTUkFtZm2BbwBPpbY4CX37QkUFLF/eXGcUEYmmpII6hLA9hNA1hLAp1QWKO+ggf/3ss+Y6o4hINEX2ycT+/f1VQS0i2S6yQV1cDPn5sGhRuksiIpJekQ3qnBzo1081ahGRyAY1ePOHglpEsl2kg/qggzyoQ0h3SURE0ifSQd2/P+zYAatWpbskIiLpE/mgBt1QFJHslhFBrXZqEclmkQ7q3r0hL09BLSLZLdJBnZcHffooqEUku0U6qMF7fqiNWkSyWeSDOt6XWl30RCRbZURQb9oE69enuyQiIumREUENaqcWkeyloBYRibjIB3XfvmCmG4oikr2iE9RVVXDSSXDnnTXezs+HXr1UoxaR7JWX7gLslpMDn3ySaOuoRqPoiUg2i06NGny2gBUrvva2glpEslm0grpnzzpns+3fH9at8256IiLZJtlZyDuZ2RNmNt/M5pnZMSkpTXHxHoMadENRRLJTsjXq3wMvhxAOAYYC81JSmp49YcMGH4S6mnhQf/ppSs4qIhJpDQa1mXUARgH3AYQQdoUQNqakNMXF/lqrnXrAAGjdGmbNSslZRUQiLZkadT9gLfCAmX1gZveaWbvaG5nZpWY2w8xmrF27tnGl6dnTX2sFdX4+DB8O//534w4rIpLJkgnqPGA48IcQwjBgG/DT2huFEO4JIZSGEEqLiooaV5p4jbqOdupjjoEZM6C8vHGHFhHJVMkE9XJgeQjhvdjvT+DB3fT2UKMGOPpoKCuDDz9MyZlFRCKrwaAOIawGlpnZwNhbY4BPUlKa9u2hY8c91qgB3n03JWcWEYmsZHt9XAE8bGYfASXAr1JWoj089FJcDAccoHZqEck+ST1CHkKYDZSmuCxuDw+9mHmtWjVqEck20XoyEfb40At4O/XixbBmTTOXSUQkjaIX1D17wurVUFHxtVVqpxaRbBS9oC4u9iFPV6/+2qrhw31mcrVTi0g2iV5Q19NFr00bGDZMNWoRyS7RC+p6HnoBb6d+//06W0ZERFqk6AV1PTVq8Hbq7dthzpxmLJOISBpFL6i7dvXBPeqpUYPaqUUke0QvqM32+NALQJ8+sN9+aqcWkewRvaCGPT70Ap7jxx0Hr74KlZXNXC4RkTSIZlDXU6MGOO88WLUKXnutGcskIpIm0QzqeI06hDpXjx8PnTvDn/7UzOUSEUmDaAZ1cTHs3Anr19e5Oj8fzj8fnn5aE96KSMsXzaBuoIsewKRJPj71Y481U5lERNIkmkHdwEMvAKWlcOihav4QkZYvmkGdRI3azGvV//oXLFzYTOUSEUmDaAb1/vtDTk69NWqACy/0zf7852Yql4hIGkQzqPPyoEePemvU4BXvb3zDg7qqqpnKJiLSzKIZ1FDvQy/VTZ4MX3wBL7+c+iKJiKRDdIO6gYde4s44A3r1gl/9ao/drkVEMlq0g3rp0gafE2/dGq67zm8qTpvWTGUTEWlGSQW1mS0xs4/NbLaZzUh1oQAfz3TrVnjvvQY3/d73fKCmKVOaoVwiIs1sb2rUJ4YQSkIIzTMb+Smn+E3FZ59tcNM2beDHP/aBmt5/vxnKJiLSjKLb9NGxI4weDc89l9Tml13m43+oVi0iLU2yQR2Av5vZTDO7tK4NzOxSM5thZjPWrl3bNKWbOBHmz4dPP21w08JCuOoqz/WPP26a04uIREGyQT0yhDAcOAX4oZmNqr1BCOGeEEJpCKG0qKioaUo3YYK/JtH8AXDFFR7YP/2peoCISMuRVFCHEFbGXtcATwNHpbJQux14IJSUJB3UXbrArbfCiy9qDBARaTkaDGoza2dmhfGfgZOB5ptaduJEeOcdSLI55Yor4IQTvBnkiy9SXDYRkWaQTI16P+CfZvYh8D7wQgih+Z4DnDjR2zGefz6pzXNy4IEHvPv1976nJhARyXwNBnUIYXEIYWhsOTyE0Lz9KkpK/NHDJJs/APr2hf/+b5+q6+67U1g2EZFmEN3ueXFmflPx73+H7duT3u3SS+Hkk+Haa2HWrBSWT0QkxaIf1OBBvWOHh3WSzPyGYlERnHoqLFmSuuKJiKRSZgT1iSfCAQfAH/6wV7vtv7/3ACkrg3Hj4KuvUlQ+EZEUyoygbtUKfvhDr1HPnbtXux52GDzzDCxaBKef7nPmiohkkswIavBG54ICmDp1r3c94QR48EEfXe+ssxTWIpJZMieou3Xzubf+/GdYv36vdz/vPLjrLu/lN3GiN3mLiGSCzAlq8KdYysrgnnsatfvll8O993oLyqmnwrZtTVw+EZEUyKygHjQIxo6F//t/oby8UYf43ve8Uv7WW959r6nGjxIRSZXMCmqAq6+GlSvhiScafYgLL4S//tX7Vx91FMxpvgfiRUT2WuYF9SmnwMEHN+qmYnVnneW16p07fTKZJJ9QFxFpdpkX1Dk53gPk3XdhwYJ9OtRRR8H06TBwoD9T87OfqUeIiERP5gU1wAUXeGA3wVimPXt6t71LLoFf/9rD+6OPmqCMIiJNJDODukcP+OY34aGHGpylPBlt23pvkOeeg9Wr4cgj4eabYfPmfS+qiMi+ysygBpg8GZYvh3/8o8kOedppfmPx9NPhv/7LR+G77TbYsqXJTiEistcyN6gnTIBOnZp8KpeiIu8RMn2632T8+c89sG+5pVHP2YiI7LPMDeqCAjjnHHjqqZS0UZSWek+Qd9/1wL7pJp8Z7KqrYMWKJj+diMgeZW5Qgzd/7NixT32qGzJiBPztbz6z+dln+2PoAwbAlCn+kKSISKpldlCPGOGp+eCDKT/VoEF+mk8/hW99C264wUfme+opqKpK+elFJItldlCbwaRJ8Pbbzdanrm9fePJJn+arbVs480wYOhT+53+goqJZiiAiWSazgxrgP/4D9tsPvvOdZu2eMWYMzJ7tPQSrqrxr94AB3ob90EMwb16T9BwUEUk+qM0s18w+MLNoPWzdvTs88ggsXOhPLDbjtON5eT5uyMcf++QEfft6f+yLL/Zmke7d4aKL4LHHYNOmZiuWiLQweXux7VXAPKBDisrSeCeeCL/8pfelO+44nw2mGeXk+BjXEyd6LXr+fO/e9/rrPhXYX/4Cubn+qHpJiTeVDB/uD9Z07NisRRWRDGQhiRqomRUDfwKmANeEEMbXt31paWmYMWNG05QwWVVVidnK//lPfxY8Aior4d//hlde8aaSDz+EZct8nRkccgiMHOk9Sk46yWvpIpJ9zGxmCKG0znVJBvUTwG1AIXBtXUFtZpcClwIceOCBRyxdunSfCt0oGzZ4VTUnx1OxQ/Qq/+DFnDkT3nvPl2nTvCt49+7eNfyUU+Doo6Fz53SXVESayz4FtZmNB8aFEH5gZqPZQ1BXl5Yaddw778Dxx8P55/tdvQxQVgYvveQ9R/72t8QIfocd5j0QBw/25bDD/P2tW/2+ac+ePtO6iGS+fQ3q24CLgAqgAG+jfiqEcOGe9klrUIMP1HHzzfDwwx7YGWTbNnj/ff++eecdb+ve0yw0ubneLn7ZZd4LJSfz+/CIZK19bvqodqDRRL1GDd6hefRo744xe7Z3x8hga9b4R1mwwNuw27eHdu08yO+/H9atgz59/CZl376+DBjgNfBevbwtXESiLfuCGmDJEk+ugw6Ca66BUaN8sI4WpqzMH8B5/HH47DP4/HPYvj2xvn17nxCne3dfioo81A8+GPr3h969oVWrtBVfRGKaLKiTFYmgBu/c/N3vwsaN/nvv3t7X+uqr/bHCFigEr4F/+il88okvn33m761d6687dtTcp2NH6NbNQ/zww2HYMO9GeOSR0Lp1ej6HSLbJ3qAG7x83Z453rXjhBe8nV1zsoypdeGHWNeyGAF9+6eG9cKF3FVy/3ptPVq3yJpZ163zbc8/1Z4lEJPWyO6hrmzYNrr3W79J16uRBXVHhjb8/+YkvubnpLmXahODDuP7iF95pZs0adRMUaQ4K6tqqqvy57jff9IBu1crbCl580W9CPvSQ17qz2PTp/szQAw/4aLIikloK6mSE4OOYXnGFN8xedZWH9X77eYflQw/1yQqyRAjee2TQIJ9AQURSq76gzq4G2vqY+Y3H2bM9lG++2UfmO+00f9qxfXt/6uSii7xfXG1VVX7zsoXMiGsGZ53lT+RrQCmR9FJQ19a/P/zrX97H7fPPfS6uxx6Dn/3Me428/HKieSRu61YfmPrb3/YeJS3EWWdBebnPzi4i6aOmj7311Vceym+84RMpTprkjwfOnev92aZP95GXBg1Kd0n3WQj+3TRsGDz7bLpLI9KyqemjKXXu7LXqyZP9UfVDDoEvvvDBOl54AQoLfbjVFsDMv5NeeaXFtOiIZCQFdWO0bu3Pbv/qV16Lfu89OPlk6NoVfvpTH1np7bdr7rN9e7NOatBUzj7bB4nSDUWR9FHTR1Pbvt2fz+7d29u6v/wSrr8e/vxn7z0yapSP7jdsGPTr548DRngwjqoqHy/kqKPg6afTXRqRlqu+pg8NU9/U2rb1HiOXXgqXXOIDcezcCT/4gT8C+OabNR/3a9vWk7B9e/+5XTvf9rTT0vUJasjJ8eaPe+6BRYt86BQRaV6qUadCRYXfTFywAMaNgzvu8Fo2ePPH4sU+CMfnn/uyYoXXxOM9Tb74Au67r+aTJqtWeVpOmuSjKjWjefN8hrMQvAPM2LHNenqRrFBfjZoQQpMvRxxxRMh6CxeG8MYbe7/f1q0hfOMbIUAIv/tdCOXl/lpY6O8VF4ewYEFi+6qqEKZODeGgg0KYMePrx1uyJIQHHvDt9sGiRSEcfngIubkh3HHHPh9ORGoBZoQ9ZKqCOvcxhWIAAA4ISURBVIrKykI480z/z9Orl79+61shPPtsCEVFIey3Xwgff+yhfv75vj4vz8N606bEcb76KoQBA3z9XXd9/TyrV4cwZ07Sxdq8OYSJE/1wo0eH8OST/j0iIvtOQZ2JystDuOyyEPr3D+GppxJV2E8+CaFHjxC6dvUqrlkIv/xlCG+9FUJOjgd3VVUIFRUhnHKKB/jw4SEUFNQM5SVL/EsgPz+E995LuliVlV7BP/DARAX/xhtDePPNELZvb+JrIJJF6gtqtVFnos8+87m3tm71G5Mnn+zv//KXcOON3nVwwQL4zW/g7rv9gZwhQ6BHD+9KuGGD9z5Zv94nAC4vhxkz4IADki5CRYV3G7/zTnjtNW+/bt0aSku9w0vXrtCli5/yoIN8OfBAzbIusicalKkl2rzZx9quPgZpZaWH9j//Cbt2weWXw113+boXX4RTT/XxTN591weifu0172ly7LE+vslbb0GbNr7/b34DxxyT1MM7Gzd6T8Rp03wYlNWr/Ttg48aaXcdbtfLhUkpL4YgjPLy7dk2Eetu2ke6pKJJSCupssmqV99E+5BAfUan6FC1XXQVTp3oYv/QSnHCCv//ss3D66d4lcOtWfzwe/EtgzZpGV4MrK704ixb5smABzJoFM2f6k/i1tW7tgd2li884E1+Kiny29R49/PWAA/znLBrMULKAgjrbbN7s1dPaAVtW5vNHnnUWnHRSzXVTpsANN3gSXn+9J+RFF3lojx7dpMULwXshxmeX2bDBX7/6KvFzfNaZdet8CrGqqq8fp1Mnn0bMzPt7t2rlob7ffv4xunb175r4Ur32XlHhM75v2+b79evn318i6aKgloaF4O0Ww4d7Ym3d6mF9+eXwu9+ltWiVlR7Yq1d7DT2+rFwJW7Z40UPw54rWrvWHQVevrrvWXp8DDvAu6nl5fs7KSp/sp3VryM/37774F8F++3mb+0EH+bjdddXuQ/AWqNxctc1Lw/YpqM2sAJgG5ONPMj4RQripvn0U1C3E+PH+YM6iRRnZeFxZ6X9c1K6pb9jgteh27fyB0O3bE80zS5d6wObm+lJV5V8Au3Z57XvNGv/SqP6/jZnX2KuqfKms9O3LyxPbtG/v23To4PvGvwgKCvy9jh39L4SiIl+6d/e/CuJLu3Zezm3b/LiHHKLZ41uafX2EfCdwUghhq5m1Av5pZi+FEN5t0lJK9Eyc6F075szxu4AZJjc30ezRr1/THbeiwmvuS5YkAn7NGj9fTk7NWnh+vgfrV1/5snlzYpvcXG+N2rzZj/fpp/6azEiFHTr4feNx43xcsP3282adLJurOWs0GNSx/n1bY7+2ii2ZNwyc7L3x4/312WczMqhTJS/Pb2b26OEdY5pa7SacVau8Jt2unS9VVT5kzIsvwhNPJPbLzfWaeM+ePotccbFvn5PjtX4zr83H2/s7dUrcrG3Txs9bVuZ/DbRpkxh6pqrKW8LitflOnbydP97hqKzMl9xcbw6qPc5YWZkfO/4FFX9vxw7/K6GqKlG+Vq38uIWFdX/phOAzDm3cmPgrpPq5Kiv9XG3aZOQfgXuUVBu1meUCM4H+wJ0hhOvr2OZS4FKAAw888IilS5c2cVElLY4+2v/1T5+e7pJILSHARx/B/Pke6mvWeLCvWAHLl/uyY4cHYTyg46Edgv9lkAr5+d7eX1bmf0WUle39MXJyPITz8xN/feza5c1O1ZuU8vL8yyY31/8S2bLF3zfzL5nCQn9t08aX1q19//Jy//yFhYkvnbw833/LFv8CyctL/FXUtq1vW1jov2/e7F8Wmzb5ceIx2rkzPPxw467bPo+eF0KoBErMrBPwtJkNCiHMqbXNPcA94G3UjSuqRM7Eid6XesUKr6pJZJjB0KG+NMa2bYleNWVlHkAFBR5QO3cmesXk5iba8/PyPJw2bPAlJ8f3KSjwIF22zMcUW7HCwy3e9FRQ4N/38S+NgoJEeObmJm4I79rlARhvKtq1y/erqEj06ikq8tp0vMlozRrft2NHbxIqKPCgrR66O3b4smuXl6tVKz/v1q1e5o8+8vDu0MHDuG1bvwabN/tr/Hjx3+O1+Q4dEj1gzerundQU9upedAhho5m9CXwLmNPA5tISxIP6uee8B4i0GPGmlN69012SzBJC8zerNHjrwcyKYjVpzKwNMBaYn+qCSUQceqhP+KsZbkWA9LR9J3OPuAfwhpl9BEwHXg0haGKmbGEGEybAP/7hU4xl4HRiIpmuwaAOIXwUQhgWQhgSQhgUQrilOQomEXL55d6FYMIE7wv2/PMKbJFmpF6X0rD+/b1rwf33+x2e007zG4vnn++zznzwgY/ot3Kl32lSiIs0KT1CLnunvNzn43rxRR8HZNWqr2/TqlViFKXDDoNvf9ufztAoSiJ7pLE+JDVC8Mfp5s5NzPm4dWuiQ++qVfD++97fqn17n2wxJ8f7OG3enOhy0Lu319pHjvTBNlrSkwoiSdIs5JIaZjBwoC97smuX17yffNJf8/O9E2rnzt4x9dVXvckkXmHo1QuOP947p375pS8bN3pH2viTDqWlHvpjx3rIxzvdbtnitfmCAj9PfMg8PVctGU41akm/Xbt8sOpp03zygnfe8XDt3t0Hsejc2QM4L8+3fecdbxNPRm6uPyHRqZPvu3OnB37XronnrLt18+PHQ75PH//yOfhgfy/+18FXX/kXQJs2vl3Hjv5YW4cO+jKQfaamD2l5liyB11/3R+s6d/bALCz0EI4PWrFhQ2LAjM2bEyMltWrl+y1b5s9Zb9jg+1VWNq4sOTneJj9smE9dM3iwH3f2bPjwQ/+LIT4gRgi+3ciRPrNOcXHiue5du/yRvhUrfJ/8fP9C6dYtMZB2fFaFLl2a9HJK+imoRZJRVeXPGS9e7DX8Tz/18I6PwNSli4dp/HnkTZsSzS5Llvj0NfPmJZ4j3n9/KCnx5pl4LXzXLp+f8v336x8EIz7s3p6eSR4xwid2OOccD27JeApqkeaybZt3ZSwu9mabPdm1y2vb69cnRkzKy/PRjHr29L8SQvD2+fhA2tW/FP76Vx+gIi/Pnx5t3dqX+CAW1cdSzctLjDAUH/y6Q4dEOeKzG3TqlBjAIoTE6EXl5YkBtquqfCCM+NB6ZWWJ0YkqKxPD8XXt6tvE7xfEX/Pza46MFD9/vFmqosLLGi979aH/cnIS6/LyEgOGxAcS2brV71Ps2JE4Z3wYwLpmbqg+mhLUHN6v+n+nbdsSI1rFyxMfULyy0m+UN8GEnwpqkZboww99qLaFCxNBumtXYvaC+ChI8aHi4oNfb9zo28Xl5/v6xjb9ZIKCAm8aa906MdpU9WH44nJz/Xrk5Xng17XNno7frZtP9zNtWqOKqF4fIi3Rvgydt3NnotYaH/d069ZE7Tg3N3GDtVWrRI3dLDHVzPbtiZuqHTt6bbP6ZJc7diTuF8QHpS4r8/CrfuzqNe34PGjxL474cHtQsxZbXl5zUOvc3MTQd23a1Gyi2rbNa9pbt/r78dGo4kP3xcUHs45/6bVpk9g2Ly/xBRifAigvzz/zli3+edev/3qNvIkoqEWyUX5+zd/NEgMu9+pV/76FhXteF59QUpqU+hSJiEScglpEJOIU1CIiEaegFhGJOAW1iEjEKahFRCJOQS0iEnEKahGRiEvJI+RmthZY2sjduwHrmrA4mU7X4+t0TWrS9agpU69H7xBCUV0rUhLU+8LMZuzpefdspOvxdbomNel61NQSr4eaPkREIk5BLSIScVEM6nvSXYCI0fX4Ol2TmnQ9ampx1yNybdQiIlJTFGvUIiJSjYJaRCTiIhPUZvYtM1tgZp+Z2U/TXZ50MLNeZvaGmc0zs7lmdlXs/S5m9qqZLYy9dk53WZuTmeWa2Qdm9nzs96y9HmbWycyeMLP5sX8nx2Tz9QAwsx/F/n+ZY2aPmFlBS7smkQhqM8sF7gROAQ4DzjOzw9JbqrSoAH4cQjgUOBr4Yew6/BR4PYRwMPB67PdschUwr9rv2Xw9fg+8HEI4BBiKX5esvR5m1hO4EigNIQwCcoFzaWHXJBJBDRwFfBZCWBxC2AU8CkxMc5maXQhhVQhhVuznLfj/hD3xa/Gn2GZ/Ak5PTwmbn5kVA6cC91Z7Oyuvh5l1AEYB9wGEEHaFEDaSpdejmjygjZnlAW2BlbSwaxKVoO4JLKv2+/LYe1nLzPoAw4D3gP1CCKvAwxzonr6SNbs7gOuAqmrvZev16AesBR6INQXda2btyN7rQQhhBXA78AWwCtgUQvg7LeyaRCWorY73srbfoJm1B54Erg4hbE53edLFzMYDa0IIM9NdlojIA4YDfwghDAO2keF/0u+rWNvzRKAvcADQzswuTG+pml5Ugno5UH3q42L8z5esY2at8JB+OITwVOztL82sR2x9D2BNusrXzEYCE8xsCd4cdpKZ/YXsvR7LgeUhhPdivz+BB3e2Xg+AscDnIYS1IYRy4CngWFrYNYlKUE8HDjazvmbWGr8Z8Fyay9TszMzw9sd5IYT/U23Vc8Ck2M+TgGebu2zpEEL4WQihOITQB/838Y8QwoVk7/VYDSwzs4Gxt8YAn5Cl1yPmC+BoM2sb+/9nDH5vp0Vdk8g8mWhm4/D2yFzg/hDClDQXqdmZ2XHA28DHJNpkf463Uz8GHIj/wzw7hLAhLYVMEzMbDVwbQhhvZl3J0uthZiX4jdXWwGLgu3iFKyuvB4CZ/RdwDt5r6gPgP4D2tKBrEpmgFhGRukWl6UNERPZAQS0iEnEKahGRiFNQi4hEnIJaRCTiFNQiIhGnoBYRibj/D0r52eEao+KIAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "#Plot Loss over epochs\n",
    "epcs = np.arange(90)\n",
    "\n",
    "plt.title('Loss over epochs')\n",
    "plt.plot(epcs,history['train_loss'],'-b',label='Train Loss')\n",
    "plt.plot(epcs,history['validation_loss'],'-r',label='Validation Loss')\n",
    "plt.legend(loc=\"upper right\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
